maride/guacamole-spice-protocol
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

GUAC-240: Implement adaptive encoding. Build a heat map of the screen, and use lossy image compression for areas refreshing frequently. Once refresh frequency is reduced the lossy area is repainted with a lossless image.

Browse Source
This commit is contained in:
Frode Langelo 2015-08-10 09:12:29 -07:00 committed by Michael Jumper
parent fde6abe98b
commit 379c4462ca
5 changed files with 864 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/common/Makefile.am
View File

@ -25,29 +25,31 @@ ACLOCAL_AMFLAGS = -I m4
noinst_LTLIBRARIES = libguac_common.la
noinst_HEADERS = \
guac_io.h \
guac_clipboard.h \
guac_dot_cursor.h \
guac_iconv.h \
guac_json.h \
guac_list.h \
guac_pointer_cursor.h \
guac_rect.h \
guac_string.h \
guac_surface.h
noinst_HEADERS = \
guac_io.h \
guac_clipboard.h \
guac_dot_cursor.h \
guac_iconv.h \
guac_json.h \
guac_list.h \
guac_pointer_cursor.h \
guac_rect.h \
guac_string.h \
guac_surface.h \
guac_surface_smoothness.h
libguac_common_la_SOURCES = \
guac_io.c \
guac_clipboard.c \
guac_dot_cursor.c \
guac_iconv.c \
guac_json.c \
guac_list.c \
guac_pointer_cursor.c \
guac_rect.c \
guac_string.c \
guac_surface.c
libguac_common_la_SOURCES = \
guac_io.c \
guac_clipboard.c \
guac_dot_cursor.c \
guac_iconv.c \
guac_json.c \
guac_list.c \
guac_pointer_cursor.c \
guac_rect.c \
guac_string.c \
guac_surface.c \
guac_surface_smoothness.c
libguac_common_la_CFLAGS = \
-Werror -Wall -pedantic \

599
src/common/guac_surface.c
View File

@ -23,15 +23,18 @@
#include "config.h"
#include "guac_rect.h"
#include "guac_surface.h"
#include "guac_surface_smoothness.h"
#include <cairo/cairo.h>
#include <guacamole/client.h>
#include <guacamole/layer.h>
#include <guacamole/protocol.h>
#include <guacamole/socket.h>
#include <guacamole/timestamp.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
/**
* The width of an update which should be considered negible and thus
@ -77,6 +80,425 @@
#define cairo_format_stride_for_width(format, width) (width*4)
#endif
/**
* The JPEG compression minimum block size. This defines the optimal rectangle
* block size factor for JPEG compression to reduce artifacts. Usually this is
* 8 (8x8), but use 16 to reduce the occurence of ringing artifacts further.
*/
#define GUAC_SURFACE_JPEG_BLOCK_SIZE 16
/**
* Minimum JPEG bitmap size (area). If the bitmap is smaller than this
* threshold, it should be compressed as a PNG image to avoid the JPEG
* compression tax.
*/
#define GUAC_SURFACE_JPEG_MIN_BITMAP_SIZE 4096
/**
* The JPEG image quality ('quantization') setting to use. Range 0-100 where
* 100 is the highest quality/largest file size, and 0 is the lowest
* quality/smallest file size.
*/
#define GUAC_SURFACE_JPEG_IMAGE_QUALITY 90
/**
* Time (msec) between each time the surface's heat map is recalculated.
*/
#define GUAC_COMMON_SURFACE_HEAT_MAP_UPDATE_FREQ 2000
/**
* Refresh frequency threshold for when an area should be refreshed lossy.
*/
#define GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY 3
/**
* Time delay threshold between two updates where a lossy area will be moved
* to the non-lossy refresh pipe.
*/
#define GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD 3000
/*
* Forward declarations.
*/
static void __guac_common_clip_rect(guac_common_surface* surface,
guac_common_rect* rect, int* sx, int* sy);
static int __guac_common_should_combine(guac_common_surface* surface,
const guac_common_rect* rect, int rect_only);
static void __guac_common_mark_dirty(guac_common_surface* surface,
const guac_common_rect* rect);
static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface,
const guac_common_rect* rect);
/**
* Flush a surface's lossy area to the dirty rectangle. This will make the
* rectangle refresh through the normal non-lossy refresh path.
*
* @param surface
* The surface whose lossy area will be moved to the dirty refresh
* queue.
*
* @param x
* The x coordinate of the area to move.
*
* @param y
* The y coordinate of the area to move.
*/
static void __guac_common_surface_flush_lossy_rect_to_dirty_rect(
guac_common_surface* surface, int x, int y) {
/* Get the heat map index. */
int hx = x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
int hy = y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
/* Don't update if this rect was not previously sent as a lossy refresh. */
if (!surface->lossy_rect[hy][hx]) {
return;
}
/* Clear the lossy status for this heat map rectangle. */
surface->lossy_rect[hy][hx] = 0;
guac_common_rect lossy_rect;
guac_common_rect_init(&lossy_rect, x, y,
GUAC_COMMON_SURFACE_HEAT_MAP_CELL, GUAC_COMMON_SURFACE_HEAT_MAP_CELL);
int sx = 0;
int sy = 0;
/* Clip operation */
__guac_common_clip_rect(surface, &lossy_rect, &sx, &sy);
if (lossy_rect.width <= 0 || lossy_rect.height <= 0)
return;
/* Flush the rectangle if not combining. */
if (!__guac_common_should_combine(surface, &lossy_rect, 0))
guac_common_surface_flush_deferred(surface);
/* Always defer draws */
__guac_common_mark_dirty(surface, &lossy_rect);
}
/**
* Actual method which flushes a bitmap described by the dirty rectangle
* on the socket associated with the surface.
*
* The bitmap will be sent as a "jpeg" or "png" instruction based on the lossy
* flag. Certain conditions may override the lossy flag and send a lossless
* update.
*
* @param surface
* The surface whose dirty area will be flushed.
*
* @param dirty_rect
* The dirty rectangle.
*
* @param lossy
* Flag indicating whether this refresh should be lossy.
*/
static void __guac_common_surface_flush_to_bitmap_impl(guac_common_surface* surface,
guac_common_rect* dirty_rect, int lossy) {
guac_socket* socket = surface->socket;
const guac_layer* layer = surface->layer;
int send_jpeg = 0;
/* Set the JPEG flag indicating whether this bitmap should be sent as JPEG.
* Only send as a JPEG if the dirty is larger than the minimum JPEG bitmap
* size to avoid the JPEG image compression tax. */
if (lossy &&
(dirty_rect->width * dirty_rect->height) > GUAC_SURFACE_JPEG_MIN_BITMAP_SIZE) {
/* Check the smoothness of the dirty rectangle. If smooth, do not send
* a JPEG as it has a higher overhead than standard PNG. */
if (!guac_common_surface_rect_is_smooth(surface, dirty_rect)) {
send_jpeg = 1;
/* Tweak the rectangle if it is to be sent as JPEG so the size
* matches the JPEG block size. */
guac_common_rect max;
guac_common_rect_init(&max, 0, 0, surface->width, surface->height);
guac_common_rect_expand_to_grid(GUAC_SURFACE_JPEG_BLOCK_SIZE,
dirty_rect, &max);
}
}
/* Get Cairo surface for specified rect.
* The buffer is created with 4 bytes per pixel because Cairo's 24 bit RGB
* really is 32 bit BGRx */
unsigned char* buffer = surface->buffer + dirty_rect->y * surface->stride + dirty_rect->x * 4;
cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24,
dirty_rect->width,
dirty_rect->height,
surface->stride);
/* Send bitmap update for the dirty rectangle */
if (send_jpeg) {
guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer,
dirty_rect->x, dirty_rect->y, rect,
GUAC_SURFACE_JPEG_IMAGE_QUALITY);
}
else {
guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, layer,
dirty_rect->x, dirty_rect->y, rect);
}
cairo_surface_destroy(rect);
}
/**
* Flushes the bitmap update currently described by a lossy rectangle within the
* given surface.
*
* Scans through the regular bitmap update queue and excludes any rectangles
* covered by the lossy rectangle.
*
* @param surface
* The surface whose lossy area will be flushed.
*/
static void __guac_common_surface_flush_lossy_bitmap(
guac_common_surface* surface) {
if (surface->lossy_dirty) {
guac_common_surface_bitmap_rect* current = surface->bitmap_queue;
int original_queue_length = surface->bitmap_queue_length;
/* Identify all bitmaps in queue which are
* covered by the lossy rectangle. */
for (int i=0; i < original_queue_length; i++) {
int intersects = guac_common_rect_intersects(&current->rect,
&surface->lossy_dirty_rect);
/* Complete intersection. */
if (intersects == 2) {
/* Exclude this from the normal refresh as it is completely
* covered by the lossy dirty rectangle. */
current->flushed = 1;
}
/* Partial intersection.
* The rectangle will be split if there is room on the queue. */
else if (intersects == 1 &&
surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE-5) {
/* Clip and split rectangle into rectangles that are outside the
* lossy rectangle which are added to the normal refresh queue.
* The remaining rectangle which overlaps with the lossy
* rectangle is marked flushed to not be refreshed in the normal
* refresh cycle.
*/
guac_common_rect split_rect;
while (guac_common_rect_clip_and_split(&current->rect,
&surface->lossy_dirty_rect, &split_rect)) {
/* Add new rectangle to update queue */
__guac_common_surface_flush_rect_to_queue(surface,
&split_rect);
}
/* Exclude the remaining part of the dirty rectangle
* which is completely covered by the lossy dirty rectangle. */
current->flushed = 1;
}
current++;
}
/* Flush the lossy bitmap */
__guac_common_surface_flush_to_bitmap_impl(surface,
&surface->lossy_dirty_rect, 1);
/* Flag this area as lossy so it can be moved back to the
* dirty rect and refreshed normally when refreshed less frequently. */
int x = surface->lossy_dirty_rect.x;
int y = surface->lossy_dirty_rect.y;
int w = (x + surface->lossy_dirty_rect.width) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
int h = (y + surface->lossy_dirty_rect.height) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
x /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
y /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
for (int j = y; j <= h; j++) {
for (int i = x; i <= w; i++) {
surface->lossy_rect[j][i] = 1;
}
}
/* Clear the lossy dirty flag. */
surface->lossy_dirty = 0;
}
}
/**
* Calculate the current average refresh frequency for a given area on the
* surface.
*
* @param surface
* The surface on which the refresh frequency will be calculated.
*
* @param x
* The x coordinate for the area.
*
* @param y
* The y coordinate for the area.
*
* @param w
* The area width.
*
* @param h
* The area height.
*
* @return
* The average refresh frequency.
*/
static unsigned int __guac_common_surface_calculate_refresh_frequency(
guac_common_surface* surface,
int x, int y, int w, int h)
{
w = (x + w) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
h = (y + h) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
x /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
y /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
unsigned int sum_frequency = 0;
unsigned int count = 0;
/* Iterate over all the heat map cells for the area
* and calculate the average refresh frequency. */
for (int hy = y; hy <= h; hy++) {
for (int hx = x; hx <= w; hx++) {
const guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx];
sum_frequency += heat_rect->frequency;
count++;
}
}
/* Calculate the average. */
if (count) {
return sum_frequency / count;
}
else {
return 0;
}
}
/**
* Update the heat map for the surface and re-calculate the refresh frequencies.
*
* Any areas of the surface which have not been updated within a given threshold
* will be moved from the lossy to the normal refresh path.
*
* @param surface
* The surface on which the heat map will be refreshed.
*
* @param now
* The current time.
*/
static void __guac_common_surface_update_heat_map(guac_common_surface* surface,
guac_timestamp now)
{
/* Only update the heat map at the given interval. */
if (now - surface->last_heat_map_update < GUAC_COMMON_SURFACE_HEAT_MAP_UPDATE_FREQ) {
return;
}
surface->last_heat_map_update = now;
const int width = surface->width / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
const int height = surface->height / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
int hx, hy;
for (hy = 0; hy < height; hy++) {
for (hx = 0; hx < width; hx++) {
guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx];
const int last_update_index = (heat_rect->index + GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ - 1) % GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ;
const guac_timestamp last_update = heat_rect->updates[last_update_index];
const guac_timestamp time_since_last = now - last_update;
/* If the time between the last 2 refreshes is larger than the
* threshold, move this rectangle back to the non-lossy
* refresh pipe. */
if (time_since_last > GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD) {
/* Send this lossy rectangle to the normal update queue. */
const int x = hx * GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
const int y = hy * GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
__guac_common_surface_flush_lossy_rect_to_dirty_rect(surface,
x, y);
/* Clear the frequency and refresh times for this square. */
heat_rect->frequency = 0;
memset(heat_rect->updates, 0, sizeof(heat_rect->updates));
continue ;
}
/* Only calculate frequency after N updates to this heat
* rectangle. */
if (heat_rect->updates[GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ - 1] == 0) {
continue;
}
/* Calculate refresh frequency. */
const guac_timestamp first_update = heat_rect->updates[heat_rect->index];
int elapsed_time = last_update - first_update;
if (elapsed_time)
heat_rect->frequency = GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ * 1000 / elapsed_time;
else
heat_rect->frequency = 0;
}
}
}
/**
* Touch the heat map with this update rectangle, so that the update
* frequency can be calculated later.
*
* @param surface
* The surface containing the rectangle to be updated.
*
* @param rect
* The rectangle updated.
*
* @param time
* The time stamp of this update.
*/
static void __guac_common_surface_touch_rect(guac_common_surface* surface,
guac_common_rect* rect, guac_timestamp time)
{
const int w = (rect->x + rect->width) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
const int h = (rect->y + rect->height) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
int hx = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
int hy = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL;
for (; hy <= h; hy++) {
for (; hx <= w; hx++) {
guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx];
heat_rect->updates[heat_rect->index] = time;
/* Move the heat index to the next. */
heat_rect->index = (heat_rect->index + 1) % GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ;
}
}
}
/**
* Updates the coordinates of the given rectangle to be within the bounds of
* the given surface.
@ -190,7 +612,7 @@ static int __guac_common_should_combine(guac_common_surface* surface, const guac
}
}
/* Otherwise, do not combine */
return 0;
@ -222,28 +644,69 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co
}
/**
* Flushes the PNG update currently described by the dirty rectangle within the
* given surface to that surface's PNG queue. There MUST be space within the
* Expands the lossy dirty rectangle of the given surface to contain the
* rectangle described by the given coordinates.
*
* @param surface
* The surface to mark as dirty.
*
* @param rect
* The rectangle of the update which is dirtying the surface.
*/
static void __guac_common_mark_lossy_dirty(guac_common_surface* surface,
const guac_common_rect* rect) {
/* Ignore empty rects */
if (rect->width <= 0 || rect->height <= 0)
return;
/* If already dirty, update existing rect */
if (surface->lossy_dirty) {
guac_common_rect_extend(&surface->lossy_dirty_rect, rect);
}
/* Otherwise init lossy dirty rect */
else {
surface->lossy_dirty_rect = *rect;
surface->lossy_dirty = 1;
}
}
/**
* Flushes the rectangle to the given surface's bitmap queue. There MUST be
* space within the queue.
*
* @param surface The surface queue to flush to.
* @param rect The rectangle to flush.
*/
static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface,
const guac_common_rect* rect) {
guac_common_surface_bitmap_rect* bitmap_rect;
/* Add new rect to queue */
bitmap_rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]);
bitmap_rect->rect = *rect;
bitmap_rect->flushed = 0;
}
/**
* Flushes the bitmap update currently described by the dirty rectangle within the
* given surface to that surface's bitmap queue. There MUST be space within the
* queue.
*
* @param surface The surface to flush.
*/
static void __guac_common_surface_flush_to_queue(guac_common_surface* surface) {
guac_common_surface_png_rect* rect;
/* Do not flush if not dirty */
if (!surface->dirty)
return;
/* Add new rect to queue */
rect = &(surface->png_queue[surface->png_queue_length++]);
rect->rect = surface->dirty_rect;
rect->flushed = 0;
__guac_common_surface_flush_rect_to_queue(surface, &surface->dirty_rect);
/* Surface now flushed */
surface->dirty = 0;
}
void guac_common_surface_flush_deferred(guac_common_surface* surface) {
@ -254,7 +717,7 @@ void guac_common_surface_flush_deferred(guac_common_surface* surface) {
/* Flush if queue size has reached maximum (space is reserved for the final dirty rect,
* as guac_common_surface_flush() MAY add an additional rect to the queue */
if (surface->png_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1)
if (surface->bitmap_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1)
guac_common_surface_flush(surface);
/* Append dirty rect to queue */
@ -672,7 +1135,7 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client,
surface->width = w;
surface->height = h;
surface->dirty = 0;
surface->png_queue_length = 0;
surface->bitmap_queue_length = 0;
/* Create corresponding Cairo surface */
surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w);
@ -691,6 +1154,22 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client,
else
surface->realized = 0;
/* Initialize heat map and adaptive coding bits. */
surface->lossy_dirty = 0;
surface->last_heat_map_update = 0;
for (int y = 0; y < GUAC_COMMON_SURFACE_HEAT_MAP_ROWS; y++) {
for (int x = 0; x < GUAC_COMMON_SURFACE_HEAT_MAP_COLS; x++) {
guac_common_surface_heat_rect *rect= & surface->heat_map[y][x];
memset(rect->updates, 0, sizeof(rect->updates));
rect->frequency = 0;
rect->index = 0;
surface->lossy_rect[y][x] = 0;
}
}
return surface;
}
@ -773,12 +1252,30 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_
if (rect.width <= 0 || rect.height <= 0)
return;
/* Flush if not combining */
if (!__guac_common_should_combine(surface, &rect, 0))
guac_common_surface_flush_deferred(surface);
unsigned int freq = 0;
/* Always defer draws */
__guac_common_mark_dirty(surface, &rect);
/* Update the heat map for the update rectangle. */
guac_timestamp time = guac_timestamp_current();
__guac_common_surface_touch_rect(surface, &rect, time);
/* Calculate the update frequency for this rectangle. */
freq = __guac_common_surface_calculate_refresh_frequency(surface, x, y, w, h);
/* If this rectangle is hot, mark lossy dirty rectangle. */
if (freq >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY) {
__guac_common_mark_lossy_dirty(surface, &rect);
}
/* Standard refresh path */
else {
/* Flush if not combining */
if (!__guac_common_should_combine(surface, &rect, 0))
guac_common_surface_flush_deferred(surface);
/* Always defer draws */
__guac_common_mark_dirty(surface, &rect);
}
}
@ -948,30 +1445,25 @@ void guac_common_surface_reset_clip(guac_common_surface* surface) {
}
/**
* Flushes the PNG update currently described by the dirty rectangle within the
* given surface directly to a "png" instruction, which is sent on the socket
* associated with the surface.
* Flushes the bitmap update currently described by the dirty rectangle within the
* given surface.
*
* @param surface The surface to flush.
*/
static void __guac_common_surface_flush_to_png(guac_common_surface* surface) {
static void __guac_common_surface_flush_to_bitmap(guac_common_surface* surface) {
if (surface->dirty) {
guac_socket* socket = surface->socket;
const guac_layer* layer = surface->layer;
guac_common_rect dirty_rect;
guac_common_rect_init(&dirty_rect,
surface->dirty_rect.x,
surface->dirty_rect.y,
surface->dirty_rect.width,
surface->dirty_rect.height);
/* Get Cairo surface for specified rect */
unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4;
cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24,
surface->dirty_rect.width,
surface->dirty_rect.height,
surface->stride);
/* Flush bitmap */
__guac_common_surface_flush_to_bitmap_impl(surface, &dirty_rect, 0);
/* Send PNG for rect */
guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER,
layer, surface->dirty_rect.x, surface->dirty_rect.y, rect);
cairo_surface_destroy(rect);
surface->realized = 1;
/* Surface is no longer dirty */
@ -982,15 +1474,15 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) {
}
/**
* Comparator for instances of guac_common_surface_png_rect, the elements
* which make up a surface's PNG buffer.
* Comparator for instances of guac_common_surface_bitmap_rect, the elements
* which make up a surface's bitmap buffer.
*
* @see qsort
*/
static int __guac_common_surface_png_rect_compare(const void* a, const void* b) {
static int __guac_common_surface_bitmap_rect_compare(const void* a, const void* b) {
guac_common_surface_png_rect* ra = (guac_common_surface_png_rect*) a;
guac_common_surface_png_rect* rb = (guac_common_surface_png_rect*) b;
guac_common_surface_bitmap_rect* ra = (guac_common_surface_bitmap_rect*) a;
guac_common_surface_bitmap_rect* rb = (guac_common_surface_bitmap_rect*) b;
/* Order roughly top to bottom, left to right */
if (ra->rect.y != rb->rect.y) return ra->rect.y - rb->rect.y;
@ -1006,31 +1498,38 @@ static int __guac_common_surface_png_rect_compare(const void* a, const void* b)
void guac_common_surface_flush(guac_common_surface* surface) {
guac_common_surface_png_rect* current = surface->png_queue;
/* Update heat map. */
guac_timestamp time = guac_timestamp_current();
__guac_common_surface_update_heat_map(surface, time);
/* Flush final dirty rectangle to queue. */
__guac_common_surface_flush_to_queue(surface);
/* Flush the lossy bitmap to client. */
__guac_common_surface_flush_lossy_bitmap(surface);
guac_common_surface_bitmap_rect* current = surface->bitmap_queue;
int i, j;
int original_queue_length;
int flushed = 0;
/* Flush final dirty rect to queue */
__guac_common_surface_flush_to_queue(surface);
original_queue_length = surface->png_queue_length;
original_queue_length = surface->bitmap_queue_length;
/* Sort updates to make combination less costly */
qsort(surface->png_queue, surface->png_queue_length, sizeof(guac_common_surface_png_rect),
__guac_common_surface_png_rect_compare);
qsort(surface->bitmap_queue, surface->bitmap_queue_length, sizeof(guac_common_surface_bitmap_rect),
__guac_common_surface_bitmap_rect_compare);
/* Flush all rects in queue */
for (i=0; i < surface->png_queue_length; i++) {
for (i=0; i < surface->bitmap_queue_length; i++) {
/* Get next unflushed candidate */
guac_common_surface_png_rect* candidate = current;
guac_common_surface_bitmap_rect* candidate = current;
if (!candidate->flushed) {
int combined = 0;
/* Build up rect as much as possible */
for (j=i; j < surface->png_queue_length; j++) {
for (j=i; j < surface->bitmap_queue_length; j++) {
if (!candidate->flushed) {
@ -1054,13 +1553,13 @@ void guac_common_surface_flush(guac_common_surface* surface) {
/* Re-add to queue if there's room and this update was modified or we expect others might be */
if ((combined > 1 || i < original_queue_length)
&& surface->png_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE)
&& surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE)
__guac_common_surface_flush_to_queue(surface);
/* Flush as PNG otherwise */
/* Flush as bitmap otherwise */
else {
if (surface->dirty) flushed++;
__guac_common_surface_flush_to_png(surface);
__guac_common_surface_flush_to_bitmap(surface);
}
}
@ -1070,7 +1569,7 @@ void guac_common_surface_flush(guac_common_surface* surface) {
}
/* Flush complete */
surface->png_queue_length = 0;
surface->bitmap_queue_length = 0;
}

99
src/common/guac_surface.h
View File

@ -33,15 +33,69 @@
#include <guacamole/socket.h>
/**
* The maximum number of updates to allow within the PNG queue.
* The maximum number of updates to allow within the bitmap queue.
*/
#define GUAC_COMMON_SURFACE_QUEUE_SIZE 256
/**
* Representation of a PNG update, having a rectangle of image data (stored
* The maximum surface width; 2x WQXGA @ 16:10.
*/
#define GUAC_COMMON_SURFACE_MAX_WIDTH 5120
/**
* The maximum surface height; 2x WQXGA @ 16:10.
*/
#define GUAC_COMMON_SURFACE_MAX_HEIGHT 3200
/**
* Heat map square size in pixels.
*/
#define GUAC_COMMON_SURFACE_HEAT_MAP_CELL 64
/**
* Heat map number of columns.
*/
#define GUAC_COMMON_SURFACE_HEAT_MAP_COLS (GUAC_COMMON_SURFACE_MAX_WIDTH / GUAC_COMMON_SURFACE_HEAT_MAP_CELL)
/**
* Heat map number of rows.
*/
#define GUAC_COMMON_SURFACE_HEAT_MAP_ROWS (GUAC_COMMON_SURFACE_MAX_HEIGHT / GUAC_COMMON_SURFACE_HEAT_MAP_CELL)
/**
* The number of time stamps to collect to be able to calculate the refresh
* frequency for a heat map cell.
*/
#define GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ 5
/**
* Representation of a rectangle or cell in the refresh heat map. This rectangle
* is used to keep track of how often an area on a surface is refreshed.
*/
typedef struct guac_common_surface_heat_rect {
/**
* Time of the last N updates, used to calculate the refresh frequency.
*/
guac_timestamp updates[GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ];
/**
* Index of the next update slot in the updates array.
*/
int index;
/**
* The current update frequency.
*/
unsigned int frequency;
} guac_common_surface_heat_rect;
/**
* Representation of a bitmap update, having a rectangle of image data (stored
* elsewhere) and a flushed/not-flushed state.
*/
typedef struct guac_common_surface_png_rect {
typedef struct guac_common_surface_bitmap_rect {
/**
* Whether this rectangle has been flushed.
@ -49,11 +103,11 @@ typedef struct guac_common_surface_png_rect {
int flushed;
/**
* The rectangle containing the PNG update.
* The rectangle containing the bitmap update.
*/
guac_common_rect rect;
} guac_common_surface_png_rect;
} guac_common_surface_bitmap_rect;
/**
* Surface which backs a Guacamole buffer or layer, automatically
@ -123,14 +177,41 @@ typedef struct guac_common_surface {
guac_common_rect clip_rect;
/**
* The number of updates in the PNG queue.
* The number of updates in the bitmap queue.
*/
int png_queue_length;
int bitmap_queue_length;
/**
* All queued PNG updates.
* All queued bitmap updates.
*/
guac_common_surface_png_rect png_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE];
guac_common_surface_bitmap_rect bitmap_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE];
/**
* Last time the heat map was refreshed.
*/
guac_timestamp last_heat_map_update;
/**
* A heat map keeping track of the refresh frequency of
* the areas of the screen.
*/
guac_common_surface_heat_rect heat_map[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS];
/*
* Map of areas currently refreshed lossy.
*/
int lossy_rect[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS];
/**
* Non-zero if this surface's lossy area is dirty and needs to be flushed,
* 0 otherwise.
*/
int lossy_dirty;
/**
* The lossy area's dirty rectangle.
*/
guac_common_rect lossy_dirty_rect;
} guac_common_surface;

158
src/common/guac_surface_smoothness.c Normal file
View File

@ -0,0 +1,158 @@
/*
* Copyright (C) 2015 Glyptodon LLC
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
/*
* Smoothness detection from:
* QEMU VNC display driver: tight encoding
*
* From libvncserver/libvncserver/tight.c
* Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
* Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
*
* Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include "guac_surface_smoothness.h"
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
/**
* The threshold to determine an image to be smooth.
*/
#define GUAC_SURFACE_SMOOTHNESS_THRESHOLD 0
/**
* Width of sub-row when detecting image smoothness.
*/
#define GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH 7
int guac_common_surface_rect_is_smooth(guac_common_surface* surface,
guac_common_rect* rect)
{
/*
* Code to guess if the image in a given rectangle is smooth
* (by applying "gradient" filter or JPEG coder).
*/
int x, y, d, dx;
unsigned int c;
unsigned int stats[256];
int pixels = 0;
int pix, left[3];
unsigned char* buffer = surface->buffer;
int stride = surface->stride;
int w = rect->x + rect->width;
int h = rect->y + rect->height;
/* If rect is out of bounds, bail out */
if (rect->x < 0 || rect->y < 0 ||
w > surface->width || h > surface->height) {
return 0;
}
/* If rect is too small to process, bail out */
if (rect->width < GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH + 1 ||
rect->height < GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH + 1) {
return 0;
}
/* Init stats array */
memset(stats, 0, sizeof (stats));
for (y = rect->y, x = rect->x; y < h && x < w;) {
/* Scan sub-sections of the surface to determine how close the colors are
* to the previous. */
for (d = 0;
d < h - y && d < w - x - GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH;
d++) {
for (c = 0; c < 3; c++) {
unsigned int index = (y+d)*stride + (x+d)*4 + c;
left[c] = buffer[index] & 0xFF;
}
for (dx = 1; dx <= GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH; dx++) {
for (c = 0; c < 3; c++) {
unsigned int index = (y+d)*stride + (x+d+dx)*4 + c;
pix = buffer[index] & 0xFF;
stats[abs(pix - left[c])]++;
left[c] = pix;
}
++pixels;
}
}
/* Advance to next section */
if (w > h) {
x += h;
y = rect->y;
} else {
x = rect->x;
y += w;
}
}
if (pixels == 0) {
return 1;
}
/* 95% smooth or more */
if (stats[0] * 33 / pixels >= 95) {
return 1;
}
unsigned int smoothness = 0;
for (c = 1; c < 8; c++) {
smoothness += stats[c] * (c * c);
if (stats[c] == 0 || stats[c] > stats[c-1] * 2) {
return 1;
}
}
for (; c < 256; c++) {
smoothness += stats[c] * (c * c);
}
smoothness /= (pixels * 3 - stats[0]);
return smoothness <= GUAC_SURFACE_SMOOTHNESS_THRESHOLD;
}

43
src/common/guac_surface_smoothness.h Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (C) 2015 Glyptodon LLC
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef __GUAC_COMMON_SURFACE_SMOOTHNESS_H
#define __GUAC_COMMON_SURFACE_SMOOTHNESS_H
#include "guac_surface.h"
/**
* Returns the smoothness of an area on a surface.
*
* @param surface
* The surface on which the rectangle exists.
*
* @param rect
* The rectangle to check for smoothness.
*
* @return
* 1 if rectangle is smooth, zero if not.
*/
int guac_common_surface_rect_is_smooth(guac_common_surface* surface,
guac_common_rect* rect);
#endif