From e73ef09fdd5bba9f5afe18fbc86720496f01e40e Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 09:35:23 -0700 Subject: [PATCH 01/27] GUAC-240: Move PNG encoder to own file, writing directly to an img stream. --- src/libguac/Makefile.am | 2 + src/libguac/encode-png.c | 402 +++++++++++++++++++++++++++++++ src/libguac/encode-png.h | 53 ++++ src/libguac/guacamole/protocol.h | 34 +++ src/libguac/protocol.c | 27 +++ 5 files changed, 518 insertions(+) create mode 100644 src/libguac/encode-png.c create mode 100644 src/libguac/encode-png.h diff --git a/src/libguac/Makefile.am b/src/libguac/Makefile.am index e53e71a1..c059c822 100644 --- a/src/libguac/Makefile.am +++ b/src/libguac/Makefile.am @@ -64,6 +64,7 @@ libguacinc_HEADERS = \ noinst_HEADERS = \ client-handlers.h \ + encode-png.h \ palette.h \ wav_encoder.h @@ -71,6 +72,7 @@ libguac_la_SOURCES = \ audio.c \ client.c \ client-handlers.c \ + encode-png.c \ error.c \ hash.c \ instruction.c \ diff --git a/src/libguac/encode-png.c b/src/libguac/encode-png.c new file mode 100644 index 00000000..d3632f28 --- /dev/null +++ b/src/libguac/encode-png.c @@ -0,0 +1,402 @@ +/* + * Copyright (C) 2015 Glyptodon LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "config.h" + +#include "encode-png.h" +#include "error.h" +#include "palette.h" +#include "protocol.h" +#include "stream.h" + +#include +#include + +#ifdef HAVE_PNGSTRUCT_H +#include +#endif + +#include +#include +#include +#include +#include + +/** + * Data describing the current write state of PNG data. + */ +typedef struct guac_png_write_state { + + /** + * The socket over which all PNG blobs will be written. + */ + guac_socket* socket; + + /** + * The Guacamole stream to associate with each PNG blob. + */ + guac_stream* stream; + + /** + * Buffer of pending PNG data. + */ + char buffer[6048]; + + /** + * The number of bytes currently stored in the buffer. + */ + int buffer_size; + +} guac_png_write_state; + +/** + * Writes the contents of the PNG write state as a blob to its associated + * socket. + * + * @param write_state + * The write state to flush. + */ +static void guac_png_flush_data(guac_png_write_state* write_state) { + + /* Send blob */ + guac_protocol_send_blob(write_state->socket, write_state->stream, + write_state->buffer, write_state->buffer_size); + + /* Clear buffer */ + write_state->buffer_size = 0; + +} + +/** + * Appends the given PNG data to the internal buffer of the given write state, + * automatically flushing the write state as necessary. + * socket. + * + * @param write_state + * The write state to append the given data to. + * + * @param data + * The PNG data to append. + * + * @param length + * The size of the given PNG data, in bytes. + */ +static void guac_png_write_data(guac_png_write_state* write_state, + const void* data, int length) { + + const unsigned char* current = data; + + /* Append all data given */ + while (length > 0) { + + /* Calculate space remaining */ + int remaining = sizeof(write_state->buffer) - write_state->buffer_size; + + /* If no space remains, flush buffer to make room */ + if (remaining == 0) { + guac_png_flush_data(write_state); + remaining = sizeof(write_state->buffer); + } + + /* Calculate size of next block of data to append */ + int block_size = remaining; + if (block_size > length) + block_size = length; + + /* Append block */ + memcpy(write_state->buffer + write_state->buffer_size, + current, block_size); + + /* Next block */ + current += block_size; + write_state->buffer_size += block_size; + length -= block_size; + + } + +} + +/** + * Writes the given buffer of PNG data to the buffer of the given write state, + * flushing that buffer to blob instructions if necessary. This handler is + * called by Cairo when writing PNG data via + * cairo_surface_write_to_png_stream(). + * + * @param closure + * Pointer to arbitrary data passed to cairo_surface_write_to_png_stream(). + * In the case of this handler, this data will be the guac_png_write_state. + * + * @param data + * The buffer of PNG data to write. + * + * @param length + * The size of the given buffer, in bytes. + * + * @return + * A Cairo status code indicating whether the write operation succeeded. + * In the case of this handler, this will always be CAIRO_STATUS_SUCCESS. + */ +static cairo_status_t guac_png_cairo_write_handler(void* closure, + const unsigned char* data, unsigned int length) { + + guac_png_write_state* write_state = (guac_png_write_state*) closure; + + /* Append data to buffer, writing as necessary */ + guac_png_write_data(write_state, data, length); + + return CAIRO_STATUS_SUCCESS; + +} + +/** + * Implementation of guac_png_write() which uses Cairo's own PNG encoder to + * write PNG data, rather than using libpng directly. + * + * @param socket + * The socket to send PNG blobs over. + * + * @param stream + * The stream to associate with each blob. + * + * @param surface + * The Cairo surface to write to the given stream and socket as PNG blobs. + * + * @return + * Zero if the encoding operation is successful, non-zero otherwise. + */ +static int guac_png_cairo_write(guac_socket* socket, guac_stream* stream, + cairo_surface_t* surface) { + + guac_png_write_state write_state; + + /* Init write state */ + write_state.socket = socket; + write_state.stream = stream; + write_state.buffer_size = 0; + + /* Write surface as PNG */ + if (cairo_surface_write_to_png_stream(surface, + guac_png_cairo_write_handler, + &write_state) != CAIRO_STATUS_SUCCESS) { + guac_error = GUAC_STATUS_INTERNAL_ERROR; + guac_error_message = "Cairo PNG backend failed"; + return -1; + } + + /* Flush remaining PNG data */ + guac_png_flush_data(&write_state); + return 0; + +} + +/** + * Writes the given buffer of PNG data to the buffer of the given write state, + * flushing that buffer to blob instructions if necessary. This handler is + * called by libpng when writing PNG data via png_write_png(). + * + * @param png + * The PNG compression state structure associated with the write operation. + * The pointer to arbitrary data will have been set to the + * guac_png_write_state by png_set_write_fn(), and will be accessible via + * png->io_ptr or png_get_io_ptr(png), depending on the version of libpng. + * + * @param data + * The buffer of PNG data to write. + * + * @param length + * The size of the given buffer, in bytes. + */ +static void guac_png_write_handler(png_structp png, png_bytep data, + png_size_t length) { + + /* Get png buffer structure */ + guac_png_write_state* write_state; +#ifdef HAVE_PNG_GET_IO_PTR + write_state = (guac_png_write_state*) png_get_io_ptr(png); +#else + write_state = (guac_png_write_state*) png->io_ptr; +#endif + + /* Append data to buffer, writing as necessary */ + guac_png_write_data(write_state, data, length); + +} + +/** + * Flushes any PNG data within the buffer of the given write state as a blob + * instruction. If no data is within the buffer, this function has no effect. + * This handler is called by libpng when it has finished writing PNG data via + * png_write_png(). + * + * @param png + * The PNG compression state structure associated with the write operation. + * The pointer to arbitrary data will have been set to the + * guac_png_write_state by png_set_write_fn(), and will be accessible via + * png->io_ptr or png_get_io_ptr(png), depending on the version of libpng. + */ +static void guac_png_flush_handler(png_structp png) { + + /* Get png buffer structure */ + guac_png_write_state* write_state; +#ifdef HAVE_PNG_GET_IO_PTR + write_state = (guac_png_write_state*) png_get_io_ptr(png); +#else + write_state = (guac_png_write_state*) png->io_ptr; +#endif + + /* Flush buffer */ + guac_png_flush_data(write_state); + +} + +int guac_png_write(guac_socket* socket, guac_stream* stream, + cairo_surface_t* surface) { + + png_structp png; + png_infop png_info; + png_byte** png_rows; + int bpp; + + int x, y; + + guac_png_write_state write_state; + + /* Get image surface properties and data */ + cairo_format_t format = cairo_image_surface_get_format(surface); + int width = cairo_image_surface_get_width(surface); + int height = cairo_image_surface_get_height(surface); + int stride = cairo_image_surface_get_stride(surface); + unsigned char* data = cairo_image_surface_get_data(surface); + + /* If not RGB24, use Cairo PNG writer */ + if (format != CAIRO_FORMAT_RGB24 || data == NULL) + return guac_png_cairo_write(socket, stream, surface); + + /* Flush pending operations to surface */ + cairo_surface_flush(surface); + + /* Attempt to build palette */ + guac_palette* palette = guac_palette_alloc(surface); + + /* If not possible, resort to Cairo PNG writer */ + if (palette == NULL) + return guac_png_cairo_write(socket, stream, surface); + + /* Calculate BPP from palette size */ + if (palette->size <= 2) bpp = 1; + else if (palette->size <= 4) bpp = 2; + else if (palette->size <= 16) bpp = 4; + else bpp = 8; + + /* Set up PNG writer */ + png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); + if (!png) { + guac_error = GUAC_STATUS_INTERNAL_ERROR; + guac_error_message = "libpng failed to create write structure"; + return -1; + } + + png_info = png_create_info_struct(png); + if (!png_info) { + png_destroy_write_struct(&png, NULL); + guac_error = GUAC_STATUS_INTERNAL_ERROR; + guac_error_message = "libpng failed to create info structure"; + return -1; + } + + /* Set error handler */ + if (setjmp(png_jmpbuf(png))) { + png_destroy_write_struct(&png, &png_info); + guac_error = GUAC_STATUS_IO_ERROR; + guac_error_message = "libpng output error"; + return -1; + } + + /* Init write state */ + write_state.socket = socket; + write_state.stream = stream; + write_state.buffer_size = 0; + + /* Set up writer */ + png_set_write_fn(png, &write_state, + guac_png_write_handler, + guac_png_flush_handler); + + /* Copy data from surface into PNG data */ + png_rows = (png_byte**) malloc(sizeof(png_byte*) * height); + for (y=0; ycolors, palette->size); + + /* Write image */ + png_set_rows(png, png_info, png_rows); + png_write_png(png, png_info, PNG_TRANSFORM_PACKING, NULL); + + /* Finish write */ + png_destroy_write_struct(&png, &png_info); + + /* Free palette */ + guac_palette_free(palette); + + /* Free PNG data */ + for (y=0; y + +/** + * Encodes the given surface as a PNG, and sends the resulting data over the + * given stream and socket as blobs. + * + * @param socket + * The socket to send PNG blobs over. + * + * @param stream + * The stream to associate with each blob. + * + * @param surface + * The Cairo surface to write to the given stream and socket as PNG blobs. + * + * @return + * Zero if the encoding operation is successful, non-zero otherwise. + */ +int guac_png_write(guac_socket* socket, guac_stream* stream, + cairo_surface_t* surface); + +#endif + diff --git a/src/libguac/guacamole/protocol.h b/src/libguac/guacamole/protocol.h index 16ea70a3..7256e7c4 100644 --- a/src/libguac/guacamole/protocol.h +++ b/src/libguac/guacamole/protocol.h @@ -647,6 +647,40 @@ int guac_protocol_send_jpeg(guac_socket* socket, guac_composite_mode mode, const guac_layer* layer, int x, int y, cairo_surface_t* surface, int quality); +/** + * Sends an img instruction over the given guac_socket connection. + * + * If an error occurs sending the instruction, a non-zero value is + * returned, and guac_error is set appropriately. + * + * @param socket + * The guac_socket connection to use when sending the img instruction. + * + * @param stream + * The stream over which the image data will be sent. + * + * @param mode + * The composite mode to use when drawing the image over the destination + * layer. + * + * @param layer + * The destination layer. + * + * @param x + * The X coordinate of the upper-left corner of the destination rectangle + * within the destination layer, in pixels. + * + * @param y + * The Y coordinate of the upper-left corner of the destination rectangle + * within the destination layer, in pixels. + * + * @return + * Zero if the instruction was successfully sent, non-zero on error. + */ +int guac_protocol_send_img(guac_socket* socket, const guac_stream* stream, + guac_composite_mode mode, const guac_layer* layer, + const char* mimetype, int x, int y); + /** * Sends a pop instruction over the given guac_socket connection. * diff --git a/src/libguac/protocol.c b/src/libguac/protocol.c index 4d13b51d..62cdf03c 100644 --- a/src/libguac/protocol.c +++ b/src/libguac/protocol.c @@ -1400,6 +1400,33 @@ int guac_protocol_send_png(guac_socket* socket, guac_composite_mode mode, } +int guac_protocol_send_img(guac_socket* socket, const guac_stream* stream, + guac_composite_mode mode, const guac_layer* layer, + const char* mimetype, int x, int y) { + + int ret_val; + + guac_socket_instruction_begin(socket); + ret_val = + guac_socket_write_string(socket, "3.img,") + || __guac_socket_write_length_int(socket, stream->index) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, mode) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, layer->index) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_string(socket, mimetype) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, x) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, y) + || guac_socket_write_string(socket, ";"); + + guac_socket_instruction_end(socket); + return ret_val; + +} + int guac_protocol_send_pop(guac_socket* socket, const guac_layer* layer) { int ret_val; From 859f7d934051ab0e70b9e111e8969d14fe924e1b Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 15:18:12 -0700 Subject: [PATCH 02/27] GUAC-240: Associate Guacamole client with surface. --- src/common/guac_surface.c | 7 +++++-- src/common/guac_surface.h | 31 +++++++++++++++++++++++++------ src/protocols/rdp/client.c | 5 +++-- src/protocols/rdp/rdp_bitmap.c | 3 ++- src/protocols/vnc/client.c | 5 +++-- src/terminal/display.c | 4 ++-- 6 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index d2c6a435..3ef36666 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -25,6 +25,7 @@ #include "guac_surface.h" #include +#include #include #include #include @@ -660,12 +661,14 @@ static void __guac_common_surface_transfer(guac_common_surface* src, int* sx, in } -guac_common_surface* guac_common_surface_alloc(guac_socket* socket, const guac_layer* layer, int w, int h) { +guac_common_surface* guac_common_surface_alloc(guac_client* client, + guac_socket* socket, const guac_layer* layer, int w, int h) { /* Init surface */ guac_common_surface* surface = malloc(sizeof(guac_common_surface)); - surface->layer = layer; + surface->client = client; surface->socket = socket; + surface->layer = layer; surface->width = w; surface->height = h; surface->dirty = 0; diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index b5ba89f2..cacfc2d9 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -27,6 +27,7 @@ #include "guac_rect.h" #include +#include #include #include #include @@ -65,6 +66,11 @@ typedef struct guac_common_surface { */ const guac_layer* layer; + /** + * The client associated with this surface. + */ + guac_client* client; + /** * The socket to send instructions on when flushing. */ @@ -131,13 +137,26 @@ typedef struct guac_common_surface { /** * Allocates a new guac_common_surface, assigning it to the given layer. * - * @param socket The socket to send instructions on when flushing. - * @param layer The layer to associate with the new surface. - * @param w The width of the surface. - * @param h The height of the surface. - * @return A newly-allocated guac_common_surface. + * @param client + * The client associated with the given layer. + * + * @param socket + * The socket to send instructions on when flushing. + * + * @param layer + * The layer to associate with the new surface. + * + * @param w + * The width of the surface. + * + * @param h + * The height of the surface. + * + * @return + * A newly-allocated guac_common_surface. */ -guac_common_surface* guac_common_surface_alloc(guac_socket* socket, const guac_layer* layer, int w, int h); +guac_common_surface* guac_common_surface_alloc(guac_client* client, + guac_socket* socket, const guac_layer* layer, int w, int h); /** * Frees the given guac_common_surface. Beware that this will NOT free any diff --git a/src/protocols/rdp/client.c b/src/protocols/rdp/client.c index ab2c104f..ccd6776d 100644 --- a/src/protocols/rdp/client.c +++ b/src/protocols/rdp/client.c @@ -933,8 +933,9 @@ int guac_client_init(guac_client* client, int argc, char** argv) { #endif /* Create default surface */ - guac_client_data->default_surface = guac_common_surface_alloc(client->socket, GUAC_DEFAULT_LAYER, - settings->width, settings->height); + guac_client_data->default_surface = guac_common_surface_alloc(client, + client->socket, GUAC_DEFAULT_LAYER, + settings->width, settings->height); guac_client_data->current_surface = guac_client_data->default_surface; /* Send connection name */ diff --git a/src/protocols/rdp/rdp_bitmap.c b/src/protocols/rdp/rdp_bitmap.c index a3211bf4..8de2fcc1 100644 --- a/src/protocols/rdp/rdp_bitmap.c +++ b/src/protocols/rdp/rdp_bitmap.c @@ -50,7 +50,8 @@ void guac_rdp_cache_bitmap(rdpContext* context, rdpBitmap* bitmap) { /* Allocate surface */ guac_layer* buffer = guac_client_alloc_buffer(client); - guac_common_surface* surface = guac_common_surface_alloc(socket, buffer, bitmap->width, bitmap->height); + guac_common_surface* surface = guac_common_surface_alloc(client, socket, + buffer, bitmap->width, bitmap->height); /* Cache image data if present */ if (bitmap->data != NULL) { diff --git a/src/protocols/vnc/client.c b/src/protocols/vnc/client.c index e40a62dc..201cdcd8 100644 --- a/src/protocols/vnc/client.c +++ b/src/protocols/vnc/client.c @@ -549,8 +549,9 @@ int guac_client_init(guac_client* client, int argc, char** argv) { guac_protocol_send_name(client->socket, rfb_client->desktopName); /* Create default surface */ - guac_client_data->default_surface = guac_common_surface_alloc(client->socket, GUAC_DEFAULT_LAYER, - rfb_client->width, rfb_client->height); + guac_client_data->default_surface = guac_common_surface_alloc(client, + client->socket, GUAC_DEFAULT_LAYER, + rfb_client->width, rfb_client->height); return 0; } diff --git a/src/terminal/display.c b/src/terminal/display.c index c90c2882..5a3b40f5 100644 --- a/src/terminal/display.c +++ b/src/terminal/display.c @@ -276,8 +276,8 @@ guac_terminal_display* guac_terminal_display_alloc(guac_client* client, /* Create default surface */ display->display_layer = guac_client_alloc_layer(client); display->select_layer = guac_client_alloc_layer(client); - display->display_surface = guac_common_surface_alloc(client->socket, - display->display_layer, 0, 0); + display->display_surface = guac_common_surface_alloc(client, + client->socket, display->display_layer, 0, 0); /* Select layer is a child of the display layer */ guac_protocol_send_move(client->socket, display->select_layer, From 56fa7423f369b2defcc0a323b92c87aa866f8a02 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 16:25:52 -0700 Subject: [PATCH 03/27] GUAC-240: Force flush at end of PNG write. --- src/libguac/encode-png.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libguac/encode-png.c b/src/libguac/encode-png.c index d3632f28..516c1f6c 100644 --- a/src/libguac/encode-png.c +++ b/src/libguac/encode-png.c @@ -396,6 +396,8 @@ int guac_png_write(guac_socket* socket, guac_stream* stream, free(png_rows[y]); free(png_rows); + /* Ensure all data is written */ + guac_png_flush_data(&write_state); return 0; } From 9c2d7f56ce1fcaaa9d212c7260b03842de754ba7 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 16:26:24 -0700 Subject: [PATCH 04/27] GUAC-240: Add function for streaming PNG images via img instruction. Use for surface. --- src/common/guac_surface.c | 3 ++- src/libguac/client.c | 22 ++++++++++++++++++++++ src/libguac/guacamole/client.h | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 3ef36666..8780c052 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -969,7 +969,8 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { surface->stride); /* Send PNG for rect */ - guac_protocol_send_png(socket, GUAC_COMP_OVER, layer, surface->dirty_rect.x, surface->dirty_rect.y, rect); + guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, + layer, surface->dirty_rect.x, surface->dirty_rect.y, rect); cairo_surface_destroy(rect); surface->realized = 1; diff --git a/src/libguac/client.c b/src/libguac/client.c index 2bb9004d..ca1c30fe 100644 --- a/src/libguac/client.c +++ b/src/libguac/client.c @@ -24,6 +24,7 @@ #include "client.h" #include "client-handlers.h" +#include "encode-png.h" #include "error.h" #include "instruction.h" #include "layer.h" @@ -375,3 +376,24 @@ void guac_client_abort(guac_client* client, guac_protocol_status status, } +void guac_client_stream_png(guac_client* client, guac_socket* socket, + guac_composite_mode mode, const guac_layer* layer, int x, int y, + cairo_surface_t* surface) { + + /* Allocate new stream for image */ + guac_stream* stream = guac_client_alloc_stream(client); + + /* Declare stream as containing image data */ + guac_protocol_send_img(socket, stream, mode, layer, "image/png", x, y); + + /* Write PNG data */ + guac_png_write(socket, stream, surface); + + /* Terminate stream */ + guac_protocol_send_end(socket, stream); + + /* Free allocated stream */ + guac_client_free_stream(client, stream); + +} + diff --git a/src/libguac/guacamole/client.h b/src/libguac/guacamole/client.h index 6d0cc85a..b8773c10 100644 --- a/src/libguac/guacamole/client.h +++ b/src/libguac/guacamole/client.h @@ -40,6 +40,7 @@ #include "stream-types.h" #include "timestamp-types.h" +#include #include struct guac_client_info { @@ -627,6 +628,39 @@ guac_object* guac_client_alloc_object(guac_client* client); */ void guac_client_free_object(guac_client* client, guac_object* object); +/** + * Streams the image data of the given surface over an image stream ("img" + * instruction) as PNG-encoded data. The image stream will be automatically + * allocated and freed. + * + * @param client + * The Guacamole client from which the image stream should be allocated. + * + * @param socket + * The socket over which instructions associated with the image stream + * should be sent. + * + * @param mode + * The composite mode to use when rendering the image over the given layer. + * + * @param layer + * The destination layer. + * + * @param x + * The X coordinate of the upper-left corner of the destination rectangle + * within the given layer. + * + * @param y + * The Y coordinate of the upper-left corner of the destination rectangle + * within the given layer. + * + * @param surface + * A Cairo surface containing the image data to be streamed. + */ +void guac_client_stream_png(guac_client* client, guac_socket* socket, + guac_composite_mode mode, const guac_layer* layer, int x, int y, + cairo_surface_t* surface); + /** * The default Guacamole client layer, layer 0. */ From 995373e74b6b15c6b26070075a1345e913c9834a Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 20:36:57 -0700 Subject: [PATCH 05/27] GUAC-240: Move JPEG encoder to own file, writing directly to an img stream. --- src/libguac/Makefile.am | 2 + src/libguac/encode-jpeg.c | 271 ++++++++++++++++++++++++++++++++++++++ src/libguac/encode-jpeg.h | 56 ++++++++ 3 files changed, 329 insertions(+) create mode 100644 src/libguac/encode-jpeg.c create mode 100644 src/libguac/encode-jpeg.h diff --git a/src/libguac/Makefile.am b/src/libguac/Makefile.am index c059c822..7e0d1696 100644 --- a/src/libguac/Makefile.am +++ b/src/libguac/Makefile.am @@ -64,6 +64,7 @@ libguacinc_HEADERS = \ noinst_HEADERS = \ client-handlers.h \ + encode-jpeg.h \ encode-png.h \ palette.h \ wav_encoder.h @@ -72,6 +73,7 @@ libguac_la_SOURCES = \ audio.c \ client.c \ client-handlers.c \ + encode-jpeg.c \ encode-png.c \ error.c \ hash.c \ diff --git a/src/libguac/encode-jpeg.c b/src/libguac/encode-jpeg.c new file mode 100644 index 00000000..e985059d --- /dev/null +++ b/src/libguac/encode-jpeg.c @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2015 Glyptodon LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "config.h" + +#include "encode-jpeg.h" +#include "error.h" +#include "palette.h" +#include "protocol.h" +#include "stream.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Extended version of the standard libjpeg jpeg_destination_mgr struct, which + * provides access to the pointers to the output buffer and size. The values + * of this structure will be initialized by jpeg_guac_dest(). + */ +typedef struct guac_jpeg_destination_mgr { + + /** + * Original jpeg_destination_mgr structure. This MUST be the first member + * for guac_jpeg_destination_mgr to be usable as a jpeg_destination_mgr. + */ + struct jpeg_destination_mgr parent; + + /** + * The socket over which all JPEG blobs will be written. + */ + guac_socket* socket; + + /** + * The Guacamole stream to associate with each JPEG blob. + */ + guac_stream* stream; + + /** + * The output buffer. + */ + unsigned char buffer[6048]; + +} guac_jpeg_destination_mgr; + +/** + * Initializes the destination structure of the given compression structure. + * + * @param cinfo + * The compression structure whose destination structure should be + * initialized. + */ +static void guac_jpeg_init_destination(j_compress_ptr cinfo) { + + guac_jpeg_destination_mgr* dest = (guac_jpeg_destination_mgr*) cinfo->dest; + + /* Init parent destination state */ + dest->parent.next_output_byte = dest->buffer; + dest->parent.free_in_buffer = sizeof(dest->buffer); + +} + +/** + * Flushes the current output buffer associated with the given compression + * structure, as the current output buffer is full. + * + * @param cinfo + * The compression structure whose output buffer should be flushed. + * + * @return + * TRUE, always, indicating that space is now available. FALSE is returned + * only by applications that may need additional time to empty the buffer. + */ +static boolean guac_jpeg_empty_output_buffer(j_compress_ptr cinfo) { + + guac_jpeg_destination_mgr* dest = (guac_jpeg_destination_mgr*) cinfo->dest; + + /* Write blob */ + guac_protocol_send_blob(dest->socket, dest->stream, + dest->buffer, sizeof(dest->buffer)); + + /* Update destination offset */ + dest->parent.next_output_byte = dest->buffer; + dest->parent.free_in_buffer = sizeof(dest->buffer); + + return TRUE; + +} + +/** + * Flushes the final blob of JPEG data, if any, as JPEG compression is now + * complete. + * + * @param cinfo + * The compression structure associated with the now-complete JPEG + * compression operation. + */ +static void guac_jpeg_term_destination(j_compress_ptr cinfo) { + + guac_jpeg_destination_mgr* dest = (guac_jpeg_destination_mgr*) cinfo->dest; + + /* Write final blob, if any */ + if (dest->parent.free_in_buffer != sizeof(dest->buffer)) + guac_protocol_send_blob(dest->socket, dest->stream, dest->buffer, + sizeof(dest->buffer) - dest->parent.free_in_buffer); + +} + +/** + * Configures the given compression structure to use the given Guacamole stream + * for JPEG output. + * + * @param cinfo + * The libjpeg compression structure to configure. + * + * @param socket + * The Guacamole socket to use when sending blob instructions. + * + * @param stream + * The stream over which JPEG-encoded blobs of image data should be sent. + */ +static void jpeg_guac_dest(j_compress_ptr cinfo, guac_socket* socket, + guac_stream* stream) { + + guac_jpeg_destination_mgr* dest; + + /* Allocate dest from pool if not already allocated */ + if (cinfo->dest == NULL) + cinfo->dest = (struct jpeg_destination_mgr*) + (cinfo->mem->alloc_small)((j_common_ptr) cinfo, JPOOL_PERMANENT, + sizeof(guac_jpeg_destination_mgr)); + + /* Pull possibly-new destination struct from cinfo */ + dest = (guac_jpeg_destination_mgr*) cinfo->dest; + + /* Associate destination handlers */ + dest->parent.init_destination = guac_jpeg_init_destination; + dest->parent.empty_output_buffer = guac_jpeg_empty_output_buffer; + dest->parent.term_destination = guac_jpeg_term_destination; + + /* Store Guacamole-specific objects */ + dest->socket = socket; + dest->stream = stream; + +} + +int guac_jpeg_write(guac_socket* socket, guac_stream* stream, + cairo_surface_t* surface, int quality) { + + /* Get image surface properties and data */ + cairo_format_t format = cairo_image_surface_get_format(surface); + + if (format != CAIRO_FORMAT_RGB24) { + guac_error = GUAC_STATUS_INTERNAL_ERROR; + guac_error_message = + "Invalid Cairo image format. Unable to create JPEG."; + return -1; + } + + int width = cairo_image_surface_get_width(surface); + int height = cairo_image_surface_get_height(surface); + int stride = cairo_image_surface_get_stride(surface); + unsigned char* data = cairo_image_surface_get_data(surface); + + /* Flush pending operations to surface */ + cairo_surface_flush(surface); + + /* Prepare JPEG bits */ + struct jpeg_compress_struct cinfo; + struct jpeg_error_mgr jerr; + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + + /* Write JPEG directly to given stream */ + jpeg_guac_dest(&cinfo, socket, stream); + + cinfo.image_width = width; /* image width and height, in pixels */ + cinfo.image_height = height; + cinfo.arith_code = TRUE; + +#ifdef JCS_EXTENSIONS + /* The Turbo JPEG extentions allows us to use the Cairo surface + * (BRGx) as input without converting it */ + cinfo.input_components = 4; + cinfo.in_color_space = JCS_EXT_BGRX; +#else + /* Standard JPEG supports RGB as input so we will have to convert + * the contents of the Cairo surface from (BRGx) to RGB */ + cinfo.input_components = 3; + cinfo.in_color_space = JCS_RGB; + + /* Create a buffer for the write scan line which is where we will + * put the converted pixels (BRGx -> RGB) */ + int write_stride = cinfo.image_width * cinfo.input_components; + unsigned char *scanline_data = malloc(write_stride); + memset(scanline_data, 0, write_stride); +#endif + + /* Initialize the JPEG compressor */ + jpeg_set_defaults(&cinfo); + jpeg_set_quality(&cinfo, quality, TRUE /* limit to baseline-JPEG values */); + jpeg_start_compress(&cinfo, TRUE); + + JSAMPROW row_pointer[1]; /* pointer to a single row */ + + /* Write scanlines to be used in JPEG compression */ + while (cinfo.next_scanline < cinfo.image_height) { + + int row_offset = stride * cinfo.next_scanline; + +#ifdef JCS_EXTENSIONS + /* In Turbo JPEG we can use the raw BGRx scanline */ + row_pointer[0] = &data[row_offset]; +#else + /* For standard JPEG libraries we have to convert the + * scanline from 24 bit (4 byte) BGRx to 24 bit (3 byte) RGB */ + unsigned char *inptr = data + row_offset; + unsigned char *outptr = scanline_data; + + for (int x = 0; x < width; ++x) { + + outptr[2] = *inptr++; /* B */ + outptr[1] = *inptr++; /* G */ + outptr[0] = *inptr++; /* R */ + inptr++; /* skip the upper byte (x/A) */ + outptr += 3; + + } + + row_pointer[0] = scanline_data; +#endif + + jpeg_write_scanlines(&cinfo, row_pointer, 1); + } + +#ifndef JCS_EXTENSIONS + free(scanline_data); +#endif + + /* Finalize compression */ + jpeg_finish_compress(&cinfo); + + /* Clean up */ + jpeg_destroy_compress(&cinfo); + return 0; + +} + diff --git a/src/libguac/encode-jpeg.h b/src/libguac/encode-jpeg.h new file mode 100644 index 00000000..204d127b --- /dev/null +++ b/src/libguac/encode-jpeg.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2015 Glyptodon LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef GUAC_ENCODE_JPEG_H +#define GUAC_ENCODE_JPEG_H + +#include "config.h" + +#include "socket.h" +#include "stream.h" + +#include + +/** + * Encodes the given surface as a JPEG, and sends the resulting data over the + * given stream and socket as blobs. + * + * @param socket + * The socket to send JPEG blobs over. + * + * @param stream + * The stream to associate with each blob. + * + * @param surface + * The Cairo surface to write to the given stream and socket as JPEG blobs. + * + * @param quality + * JPEG image quality. + * + * @return + * Zero if the encoding operation is successful, non-zero otherwise. + */ +int guac_jpeg_write(guac_socket* socket, guac_stream* stream, + cairo_surface_t* surface, int quality); + +#endif + From 1263965511a3ba55dd5ad1f3e3c35240c562eca9 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 20:43:46 -0700 Subject: [PATCH 06/27] GUAC-240: Add function for streaming JPEG images via img instruction. --- src/libguac/client.c | 22 ++++++++++++++++++++++ src/libguac/guacamole/client.h | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/libguac/client.c b/src/libguac/client.c index ca1c30fe..f7747ea3 100644 --- a/src/libguac/client.c +++ b/src/libguac/client.c @@ -24,6 +24,7 @@ #include "client.h" #include "client-handlers.h" +#include "encode-jpeg.h" #include "encode-png.h" #include "error.h" #include "instruction.h" @@ -397,3 +398,24 @@ void guac_client_stream_png(guac_client* client, guac_socket* socket, } +void guac_client_stream_jpeg(guac_client* client, guac_socket* socket, + guac_composite_mode mode, const guac_layer* layer, int x, int y, + cairo_surface_t* surface, int quality) { + + /* Allocate new stream for image */ + guac_stream* stream = guac_client_alloc_stream(client); + + /* Declare stream as containing image data */ + guac_protocol_send_img(socket, stream, mode, layer, "image/jpeg", x, y); + + /* Write JPEG data */ + guac_jpeg_write(socket, stream, surface, quality); + + /* Terminate stream */ + guac_protocol_send_end(socket, stream); + + /* Free allocated stream */ + guac_client_free_stream(client, stream); + +} + diff --git a/src/libguac/guacamole/client.h b/src/libguac/guacamole/client.h index b8773c10..3aabf2af 100644 --- a/src/libguac/guacamole/client.h +++ b/src/libguac/guacamole/client.h @@ -661,6 +661,39 @@ void guac_client_stream_png(guac_client* client, guac_socket* socket, guac_composite_mode mode, const guac_layer* layer, int x, int y, cairo_surface_t* surface); +/** + * Streams the image data of the given surface over an image stream ("img" + * instruction) as JPEG-encoded data at the given quality. The image stream + * will be automatically allocated and freed. + * + * @param client + * The Guacamole client from which the image stream should be allocated. + * + * @param socket + * The socket over which instructions associated with the image stream + * should be sent. + * + * @param mode + * The composite mode to use when rendering the image over the given layer. + * + * @param layer + * The destination layer. + * + * @param x + * The X coordinate of the upper-left corner of the destination rectangle + * within the given layer. + * + * @param y + * The Y coordinate of the upper-left corner of the destination rectangle + * within the given layer. + * + * @param surface + * A Cairo surface containing the image data to be streamed. + */ +void guac_client_stream_jpeg(guac_client* client, guac_socket* socket, + guac_composite_mode mode, const guac_layer* layer, int x, int y, + cairo_surface_t* surface, int quality); + /** * The default Guacamole client layer, layer 0. */ From 78b7b73e7853c0060652af7851637f0b350f7850 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Tue, 11 Aug 2015 20:46:43 -0700 Subject: [PATCH 07/27] GUAC-240: Remove guac_protocol_send_png() and guac_protocol_send_jpeg(). --- src/common/guac_dot_cursor.c | 3 +- src/common/guac_pointer_cursor.c | 3 +- src/libguac/guacamole/protocol.h | 53 --- src/libguac/protocol.c | 620 ------------------------------- src/protocols/rdp/rdp_pointer.c | 3 +- src/protocols/vnc/vnc_handlers.c | 4 +- src/terminal/ibar.c | 2 +- src/terminal/pointer.c | 2 +- 8 files changed, 10 insertions(+), 680 deletions(-) diff --git a/src/common/guac_dot_cursor.c b/src/common/guac_dot_cursor.c index 559c9d80..cf672a46 100644 --- a/src/common/guac_dot_cursor.c +++ b/src/common/guac_dot_cursor.c @@ -66,7 +66,8 @@ void guac_common_set_dot_cursor(guac_client* client) { guac_common_dot_cursor_height, guac_common_dot_cursor_stride); - guac_protocol_send_png(socket, GUAC_COMP_SRC, cursor, 0, 0, graphic); + guac_client_stream_png(client, socket, GUAC_COMP_SRC, cursor, + 0, 0, graphic); cairo_surface_destroy(graphic); /* Set cursor */ diff --git a/src/common/guac_pointer_cursor.c b/src/common/guac_pointer_cursor.c index 5c1d2d97..65fd43f2 100644 --- a/src/common/guac_pointer_cursor.c +++ b/src/common/guac_pointer_cursor.c @@ -77,7 +77,8 @@ void guac_common_set_pointer_cursor(guac_client* client) { guac_common_pointer_cursor_height, guac_common_pointer_cursor_stride); - guac_protocol_send_png(socket, GUAC_COMP_SRC, cursor, 0, 0, graphic); + guac_client_stream_png(client, socket, GUAC_COMP_SRC, cursor, + 0, 0, graphic); cairo_surface_destroy(graphic); /* Set cursor */ diff --git a/src/libguac/guacamole/protocol.h b/src/libguac/guacamole/protocol.h index 7256e7c4..5aa7f128 100644 --- a/src/libguac/guacamole/protocol.h +++ b/src/libguac/guacamole/protocol.h @@ -594,59 +594,6 @@ int guac_protocol_send_lstroke(guac_socket* socket, guac_line_cap_style cap, guac_line_join_style join, int thickness, const guac_layer* srcl); -/** - * Sends a png instruction over the given guac_socket connection. The PNG image - * data given will be automatically base64-encoded for transmission. - * - * If an error occurs sending the instruction, a non-zero value is - * returned, and guac_error is set appropriately. - * - * @param socket The guac_socket connection to use. - * @param mode The composite mode to use. - * @param layer The destination layer. - * @param x The destination X coordinate. - * @param y The destination Y coordinate. - * @param surface A cairo surface containing the image data to send. - * @return Zero on success, non-zero on error. - */ -int guac_protocol_send_png(guac_socket* socket, guac_composite_mode mode, - const guac_layer* layer, int x, int y, cairo_surface_t* surface); - -/** - * Sends a jpeg instruction over the given guac_socket connection. The JPEG image - * data given will be automatically base64-encoded for transmission. - * - * If an error occurs sending the instruction, a non-zero value is - * returned, and guac_error is set appropriately. - * - * @param socket - * The guac_socket connection to use. - * - * @param mode - * The composite mode to use. - * - * @param layer - * The destination layer. - * - * @param x - * The destination X coordinate. - * - * @param y - * The destination Y coordinate. - * - * @param surface - * A cairo surface containing the image data to send. - * - * @param quality - * JPEG image quality. - * - * @return - * Zero on success, non-zero on error. - */ -int guac_protocol_send_jpeg(guac_socket* socket, guac_composite_mode mode, - const guac_layer* layer, int x, int y, cairo_surface_t* surface, - int quality); - /** * Sends an img instruction over the given guac_socket connection. * diff --git a/src/libguac/protocol.c b/src/libguac/protocol.c index 62cdf03c..00aa8bbd 100644 --- a/src/libguac/protocol.c +++ b/src/libguac/protocol.c @@ -31,13 +31,7 @@ #include "stream.h" #include "unicode.h" -#include #include -#include - -#ifdef HAVE_PNGSTRUCT_H -#include -#endif #include #include @@ -75,571 +69,6 @@ ssize_t __guac_socket_write_length_double(guac_socket* socket, double d) { } -#ifndef HAVE_JPEG_MEM_DEST - -/** - * The number of bytes to allocate for the initial JPEG output buffer, if no - * buffer (or an empty buffer) is provided. - */ -#define GUAC_JPEG_DEFAULT_BUFFER_SIZE 16384 - -/** - * Extended version of the standard libjpeg jpeg_destination_mgr struct, which - * provides access to the pointers to the output buffer and size. The values - * of this structure will be initialized by jpeg_mem_dest(). - */ -typedef struct guac_jpeg_destination_mgr { - - /** - * Original jpeg_destination_mgr structure. This MUST be the first member - * for guac_jpeg_destination_mgr to be usable as a jpeg_destination_mgr. - */ - struct jpeg_destination_mgr parent; - - /** - * The current output buffer. - */ - unsigned char* buffer; - - /** - * The size of the current output buffer, in bytes. - */ - unsigned long size; - - /** - * The originally-supplied (via jpeg_mem_dest()) pointer to output buffer - * pointer. - */ - unsigned char** outbuffer; - - /** - * The originally-supplied (via jpeg_mem_dest()) pointer to the output - * buffer size, in bytes. - */ - unsigned long* outsize; - -} guac_jpeg_destination_mgr; - -/** - * Initializes the destination structure of the given compression structure. - * - * @param cinfo - * The compression structure whose destination structure should be - * initialized. - */ -static void guac_jpeg_init_destination(j_compress_ptr cinfo) { - - guac_jpeg_destination_mgr* dest = (guac_jpeg_destination_mgr*) cinfo->dest; - - /* Init parent destination state */ - dest->parent.next_output_byte = dest->buffer; - dest->parent.free_in_buffer = dest->size; - -} - -/** - * Re-allocates the output buffer associated with the given compression - * structure, as the current output buffer is full. - * - * @param cinfo - * The compression structure whose output buffer should be re-allocated. - * - * @return - * TRUE, always, indicating that space is now available. FALSE is returned - * only by applications that may need additional time to empty the buffer. - */ -static boolean guac_jpeg_empty_output_buffer(j_compress_ptr cinfo) { - - guac_jpeg_destination_mgr* dest = (guac_jpeg_destination_mgr*) cinfo->dest; - unsigned long current_offset = dest->size; - - /* Double size of current buffer */ - dest->size *= 2; - dest->buffer = realloc(dest->buffer, dest->size); - - /* Update destination offset */ - dest->parent.next_output_byte = dest->buffer + current_offset; - dest->parent.free_in_buffer = dest->size - current_offset; - - return TRUE; - -} - -/** - * Updates the given compression structure such that the originally-supplied - * output buffer size describes the size of the output image, not the overall - * size of the output buffer. This function is automatically called once JPEG - * compression ends. - * - * @param cinfo - * The compression structure associated with the now-complete JPEG - * compression operation. - */ -static void guac_jpeg_term_destination(j_compress_ptr cinfo) { - - guac_jpeg_destination_mgr* dest = (guac_jpeg_destination_mgr*) cinfo->dest; - - /* Commit current buffer and total image size to provided pointers */ - *dest->outbuffer = dest->buffer; - *dest->outsize = dest->size - dest->parent.free_in_buffer; - -} - -/** - * Configures the given compression structure to use the given buffer for JPEG - * output. If no buffer is supplied, a new buffer is allocated. This is a - * limited, internal implementation of libjpeg's jpeg_mem_dest(), which is - * unavailable in older versions of libjpeg. - * - * @param cinfo - * The libjpeg compression structure to configure. - * - * @param outbuffer - * Pointer to a pointer to the output buffer to write JPEG data to. If the - * output buffer pointer is NULL, a new output buffer will be allocated and - * assigned. - * - * @param outsize - * Pointer to the size of the output buffer, if an output buffer is - * supplied. If a new output buffer is allocated, its size will be stored - * here. When JPEG compression is complete, the total image size (NOT - * buffer size) will be stored here. - */ -static void jpeg_mem_dest(j_compress_ptr cinfo, - unsigned char** outbuffer, unsigned long* outsize) { - - guac_jpeg_destination_mgr* dest; - - /* Allocate dest from pool if not already allocated */ - if (cinfo->dest == NULL) - cinfo->dest = (struct jpeg_destination_mgr*) - (cinfo->mem->alloc_small)((j_common_ptr) cinfo, JPOOL_PERMANENT, - sizeof(guac_jpeg_destination_mgr)); - - /* Pull possibly-new destination struct from cinfo */ - dest = (guac_jpeg_destination_mgr*) cinfo->dest; - - /* Allocate output buffer if necessary */ - if (*outbuffer == NULL || *outsize == 0) { - dest->buffer = malloc(GUAC_JPEG_DEFAULT_BUFFER_SIZE); - dest->size = GUAC_JPEG_DEFAULT_BUFFER_SIZE; - } - - /* Otherwise, use provided buffer */ - else { - dest->buffer = *outbuffer; - dest->size = *outsize; - } - - /* Associate destination handlers */ - dest->parent.init_destination = guac_jpeg_init_destination; - dest->parent.empty_output_buffer = guac_jpeg_empty_output_buffer; - dest->parent.term_destination = guac_jpeg_term_destination; - - /* Store provided pointers */ - dest->outbuffer = outbuffer; - dest->outsize = outsize; - -} -#endif - -/** - * Converts the image data on a Cairo surface to a JPEG image and sends it - * as a base64-encoded transmission on the socket. - * - * @param socket - * The guac_socket connection to use. - * - * @param surface - *   A cairo surface containing the image data to send. - * - * @param quality - * JPEG image quality. - * - * @return - * Zero on success, non-zero on error. - */ -static int __guac_socket_write_length_jpeg(guac_socket* socket, - cairo_surface_t* surface, int quality) { - - /* Get image surface properties and data */ - cairo_format_t format = cairo_image_surface_get_format(surface); - - if (format != CAIRO_FORMAT_RGB24) { - guac_error = GUAC_STATUS_INTERNAL_ERROR; - guac_error_message = "Invalid Cairo image format. Unable to create JPEG."; - return -1; - } - - int width = cairo_image_surface_get_width(surface); - int height = cairo_image_surface_get_height(surface); - int stride = cairo_image_surface_get_stride(surface); - unsigned char* data = cairo_image_surface_get_data(surface); - - /* Flush pending operations to surface */ - cairo_surface_flush(surface); - - /* Prepare JPEG bits */ - struct jpeg_compress_struct cinfo; - struct jpeg_error_mgr jerr; - cinfo.err = jpeg_std_error(&jerr); - jpeg_create_compress(&cinfo); - - unsigned char *mem = NULL; - unsigned long mem_size = 0; - jpeg_mem_dest(&cinfo, &mem, &mem_size); - - cinfo.image_width = width; /* image width and height, in pixels */ - cinfo.image_height = height; - cinfo.arith_code = TRUE; - -#ifdef JCS_EXTENSIONS - /* The Turbo JPEG extentions allows us to use the Cairo surface - * (BRGx) as input without converting it */ - cinfo.input_components = 4; - cinfo.in_color_space = JCS_EXT_BGRX; -#else - /* Standard JPEG supports RGB as input so we will have to convert - * the contents of the Cairo surface from (BRGx) to RGB */ - cinfo.input_components = 3; - cinfo.in_color_space = JCS_RGB; - - /* Create a buffer for the write scan line which is where we will - * put the converted pixels (BRGx -> RGB) */ - int write_stride = cinfo.image_width * cinfo.input_components; - unsigned char *scanline_data = malloc(write_stride); - memset(scanline_data, 0, write_stride); -#endif - - /* Initialize the JPEG compressor */ - jpeg_set_defaults(&cinfo); - jpeg_set_quality(&cinfo, quality, TRUE /* limit to baseline-JPEG values */); - jpeg_start_compress(&cinfo, TRUE); - - JSAMPROW row_pointer[1]; /* pointer to a single row */ - - /* Write scanlines to be used in JPEG compression */ - while (cinfo.next_scanline < cinfo.image_height) { - - int row_offset = stride * cinfo.next_scanline; - -#ifdef JCS_EXTENSIONS - /* In Turbo JPEG we can use the raw BGRx scanline */ - row_pointer[0] = &data[row_offset]; -#else - /* For standard JPEG libraries we have to convert the - * scanline from 24 bit (4 byte) BGRx to 24 bit (3 byte) RGB */ - unsigned char *inptr = data + row_offset; - unsigned char *outptr = scanline_data; - - for (int x = 0; x < width; ++x) { - - outptr[2] = *inptr++; /* B */ - outptr[1] = *inptr++; /* G */ - outptr[0] = *inptr++; /* R */ - inptr++; /* skip the upper byte (x/A) */ - outptr += 3; - - } - - row_pointer[0] = scanline_data; -#endif - - jpeg_write_scanlines(&cinfo, row_pointer, 1); - } - -#ifndef JCS_EXTENSIONS - free(scanline_data); -#endif - - /* Finalize compression */ - jpeg_finish_compress(&cinfo); - - int base64_length = (mem_size + 2) / 3 * 4; - - /* Write length and data */ - if (guac_socket_write_int(socket, base64_length) - || guac_socket_write_string(socket, ".") - || guac_socket_write_base64(socket, mem, mem_size) - || guac_socket_flush_base64(socket)) { - - /* Something failed. Clean up and return error code. */ - jpeg_destroy_compress(&cinfo); - free(mem); - - return -1; - } - - /* Clean up */ - jpeg_destroy_compress(&cinfo); - free(mem); - - return 0; - -} - -/* PNG output formatting */ - -typedef struct __guac_socket_write_png_data { - - guac_socket* socket; - - char* buffer; - int buffer_size; - int data_size; - -} __guac_socket_write_png_data; - -cairo_status_t __guac_socket_write_png_cairo(void* closure, const unsigned char* data, unsigned int length) { - - __guac_socket_write_png_data* png_data = (__guac_socket_write_png_data*) closure; - - /* Calculate next buffer size */ - int next_size = png_data->data_size + length; - - /* If need resizing, double buffer size until big enough */ - if (next_size > png_data->buffer_size) { - - char* new_buffer; - - do { - png_data->buffer_size <<= 1; - } while (next_size > png_data->buffer_size); - - /* Resize buffer */ - new_buffer = realloc(png_data->buffer, png_data->buffer_size); - png_data->buffer = new_buffer; - - } - - /* Append data to buffer */ - memcpy(png_data->buffer + png_data->data_size, data, length); - png_data->data_size += length; - - return CAIRO_STATUS_SUCCESS; - -} - -int __guac_socket_write_length_png_cairo(guac_socket* socket, cairo_surface_t* surface) { - - __guac_socket_write_png_data png_data; - int base64_length; - - /* Write surface */ - - png_data.socket = socket; - png_data.buffer_size = 8192; - png_data.buffer = malloc(png_data.buffer_size); - png_data.data_size = 0; - - if (cairo_surface_write_to_png_stream(surface, __guac_socket_write_png_cairo, &png_data) != CAIRO_STATUS_SUCCESS) { - guac_error = GUAC_STATUS_INTERNAL_ERROR; - guac_error_message = "Cairo PNG backend failed"; - return -1; - } - - base64_length = (png_data.data_size + 2) / 3 * 4; - - /* Write length and data */ - if ( - guac_socket_write_int(socket, base64_length) - || guac_socket_write_string(socket, ".") - || guac_socket_write_base64(socket, png_data.buffer, png_data.data_size) - || guac_socket_flush_base64(socket)) { - free(png_data.buffer); - return -1; - } - - free(png_data.buffer); - return 0; - -} - -void __guac_socket_write_png(png_structp png, - png_bytep data, png_size_t length) { - - /* Get png buffer structure */ - __guac_socket_write_png_data* png_data; -#ifdef HAVE_PNG_GET_IO_PTR - png_data = (__guac_socket_write_png_data*) png_get_io_ptr(png); -#else - png_data = (__guac_socket_write_png_data*) png->io_ptr; -#endif - - /* Calculate next buffer size */ - int next_size = png_data->data_size + length; - - /* If need resizing, double buffer size until big enough */ - if (next_size > png_data->buffer_size) { - - char* new_buffer; - - do { - png_data->buffer_size <<= 1; - } while (next_size > png_data->buffer_size); - - /* Resize buffer */ - new_buffer = realloc(png_data->buffer, png_data->buffer_size); - png_data->buffer = new_buffer; - - } - - /* Append data to buffer */ - memcpy(png_data->buffer + png_data->data_size, data, length); - png_data->data_size += length; - -} - -void __guac_socket_flush_png(png_structp png) { - /* Dummy function */ -} - -int __guac_socket_write_length_png(guac_socket* socket, cairo_surface_t* surface) { - - png_structp png; - png_infop png_info; - png_byte** png_rows; - int bpp; - - int x, y; - - __guac_socket_write_png_data png_data; - int base64_length; - - /* Get image surface properties and data */ - cairo_format_t format = cairo_image_surface_get_format(surface); - int width = cairo_image_surface_get_width(surface); - int height = cairo_image_surface_get_height(surface); - int stride = cairo_image_surface_get_stride(surface); - unsigned char* data = cairo_image_surface_get_data(surface); - - /* If not RGB24, use Cairo PNG writer */ - if (format != CAIRO_FORMAT_RGB24 || data == NULL) - return __guac_socket_write_length_png_cairo(socket, surface); - - /* Flush pending operations to surface */ - cairo_surface_flush(surface); - - /* Attempt to build palette */ - guac_palette* palette = guac_palette_alloc(surface); - - /* If not possible, resort to Cairo PNG writer */ - if (palette == NULL) - return __guac_socket_write_length_png_cairo(socket, surface); - - /* Calculate BPP from palette size */ - if (palette->size <= 2) bpp = 1; - else if (palette->size <= 4) bpp = 2; - else if (palette->size <= 16) bpp = 4; - else bpp = 8; - - /* Set up PNG writer */ - png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); - if (!png) { - guac_error = GUAC_STATUS_INTERNAL_ERROR; - guac_error_message = "libpng failed to create write structure"; - return -1; - } - - png_info = png_create_info_struct(png); - if (!png_info) { - png_destroy_write_struct(&png, NULL); - guac_error = GUAC_STATUS_INTERNAL_ERROR; - guac_error_message = "libpng failed to create info structure"; - return -1; - } - - /* Set error handler */ - if (setjmp(png_jmpbuf(png))) { - png_destroy_write_struct(&png, &png_info); - guac_error = GUAC_STATUS_IO_ERROR; - guac_error_message = "libpng output error"; - return -1; - } - - /* Set up buffer structure */ - png_data.socket = socket; - png_data.buffer_size = 8192; - png_data.buffer = malloc(png_data.buffer_size); - png_data.data_size = 0; - - /* Set up writer */ - png_set_write_fn(png, &png_data, - __guac_socket_write_png, - __guac_socket_flush_png); - - /* Copy data from surface into PNG data */ - png_rows = (png_byte**) malloc(sizeof(png_byte*) * height); - for (y=0; ycolors, palette->size); - - /* Write image */ - png_set_rows(png, png_info, png_rows); - png_write_png(png, png_info, PNG_TRANSFORM_PACKING, NULL); - - /* Finish write */ - png_destroy_write_struct(&png, &png_info); - - /* Free palette */ - guac_palette_free(palette); - - /* Free PNG data */ - for (y=0; yindex) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_int(socket, x) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_int(socket, y) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_jpeg(socket, surface, quality) - || guac_socket_write_string(socket, ";"); - - guac_socket_instruction_end(socket); - return ret_val; - -} - -int guac_protocol_send_png(guac_socket* socket, guac_composite_mode mode, - const guac_layer* layer, int x, int y, cairo_surface_t* surface) { - - int ret_val; - - guac_socket_instruction_begin(socket); - ret_val = - guac_socket_write_string(socket, "3.png,") - || __guac_socket_write_length_int(socket, mode) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_int(socket, layer->index) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_int(socket, x) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_int(socket, y) - || guac_socket_write_string(socket, ",") - || __guac_socket_write_length_png(socket, surface) - || guac_socket_write_string(socket, ";"); - - guac_socket_instruction_end(socket); - return ret_val; - -} - int guac_protocol_send_img(guac_socket* socket, const guac_stream* stream, guac_composite_mode mode, const guac_layer* layer, const char* mimetype, int x, int y) { diff --git a/src/protocols/rdp/rdp_pointer.c b/src/protocols/rdp/rdp_pointer.c index 5f9c977d..1d214df2 100644 --- a/src/protocols/rdp/rdp_pointer.c +++ b/src/protocols/rdp/rdp_pointer.c @@ -60,7 +60,8 @@ void guac_rdp_pointer_new(rdpContext* context, rdpPointer* pointer) { pointer->width, pointer->height, 4*pointer->width); /* Send surface to buffer */ - guac_protocol_send_png(socket, GUAC_COMP_SRC, buffer, 0, 0, surface); + guac_client_stream_png(client, socket, GUAC_COMP_SRC, buffer, + 0, 0, surface); /* Free surface */ cairo_surface_destroy(surface); diff --git a/src/protocols/vnc/vnc_handlers.c b/src/protocols/vnc/vnc_handlers.c index 26fdef85..7e7ccee9 100644 --- a/src/protocols/vnc/vnc_handlers.c +++ b/src/protocols/vnc/vnc_handlers.c @@ -121,8 +121,8 @@ void guac_vnc_cursor(rfbClient* client, int x, int y, int w, int h, int bpp) { /* Send cursor data*/ surface = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_ARGB32, w, h, stride); - guac_protocol_send_png(socket, - GUAC_COMP_SRC, cursor_layer, 0, 0, surface); + guac_client_stream_png(gc, socket, GUAC_COMP_SRC, cursor_layer, + 0, 0, surface); /* Update cursor */ guac_protocol_send_cursor(socket, x, y, cursor_layer, 0, 0, w, h); diff --git a/src/terminal/ibar.c b/src/terminal/ibar.c index ecdcc5cb..7ba69c3b 100644 --- a/src/terminal/ibar.c +++ b/src/terminal/ibar.c @@ -78,7 +78,7 @@ guac_terminal_cursor* guac_terminal_create_ibar(guac_client* client) { guac_terminal_ibar_height, guac_terminal_ibar_stride); - guac_protocol_send_png(socket, GUAC_COMP_SRC, cursor->buffer, + guac_client_stream_png(client, socket, GUAC_COMP_SRC, cursor->buffer, 0, 0, graphic); cairo_surface_destroy(graphic); diff --git a/src/terminal/pointer.c b/src/terminal/pointer.c index 1b7ea272..72e99e3b 100644 --- a/src/terminal/pointer.c +++ b/src/terminal/pointer.c @@ -78,7 +78,7 @@ guac_terminal_cursor* guac_terminal_create_pointer(guac_client* client) { guac_terminal_pointer_height, guac_terminal_pointer_stride); - guac_protocol_send_png(socket, GUAC_COMP_SRC, cursor->buffer, + guac_client_stream_png(client, socket, GUAC_COMP_SRC, cursor->buffer, 0, 0, graphic); cairo_surface_destroy(graphic); From 29527509dec6782e2b6b04951bc6c9e727832c57 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Wed, 12 Aug 2015 16:50:24 -0700 Subject: [PATCH 08/27] GUAC-240: No longer need to test for jpeg_mem_dest(). --- configure.ac | 7 ------- 1 file changed, 7 deletions(-) diff --git a/configure.ac b/configure.ac index 653495d7..43bb38d4 100644 --- a/configure.ac +++ b/configure.ac @@ -110,13 +110,6 @@ AC_CHECK_DECL([cairo_format_stride_for_width], [Whether cairo_format_stride_for_width() is defined])],, [#include ]) -AC_CHECK_DECL([jpeg_mem_dest], - [AC_DEFINE([HAVE_JPEG_MEM_DEST],, - [Whether jpeg_mem_dest() is defined])],, - [#include - #include - #include ]) - # Typedefs AC_TYPE_SIZE_T AC_TYPE_SSIZE_T From 9edf33da2e4a857c7ab12d0eaa6cf68a2af29574 Mon Sep 17 00:00:00 2001 From: Frode Langelo Date: Mon, 10 Aug 2015 10:01:01 -0700 Subject: [PATCH 09/27] GUAC-1290: Change GUAC_VNC_FRAME_TIMEOUT to 10 so more VNC messages are handled in each frame. --- src/protocols/vnc/client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protocols/vnc/client.h b/src/protocols/vnc/client.h index ccbac69e..428ffff9 100644 --- a/src/protocols/vnc/client.h +++ b/src/protocols/vnc/client.h @@ -53,7 +53,7 @@ * milliseconds. If the server is silent for at least this amount of time, the * frame will be considered finished. */ -#define GUAC_VNC_FRAME_TIMEOUT 0 +#define GUAC_VNC_FRAME_TIMEOUT 10 /** * The number of milliseconds to wait between connection attempts. From fde6abe98b7ba5fcd0892f9d6e30e8399a4d4472 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Wed, 12 Aug 2015 21:08:44 -0700 Subject: [PATCH 10/27] GUAC-240: Byte order of pixels is BGR, not BRG. --- src/libguac/encode-jpeg.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libguac/encode-jpeg.c b/src/libguac/encode-jpeg.c index e985059d..83f6cd8c 100644 --- a/src/libguac/encode-jpeg.c +++ b/src/libguac/encode-jpeg.c @@ -203,17 +203,17 @@ int guac_jpeg_write(guac_socket* socket, guac_stream* stream, #ifdef JCS_EXTENSIONS /* The Turbo JPEG extentions allows us to use the Cairo surface - * (BRGx) as input without converting it */ + * (BGRx) as input without converting it */ cinfo.input_components = 4; cinfo.in_color_space = JCS_EXT_BGRX; #else /* Standard JPEG supports RGB as input so we will have to convert - * the contents of the Cairo surface from (BRGx) to RGB */ + * the contents of the Cairo surface from (BGRx) to RGB */ cinfo.input_components = 3; cinfo.in_color_space = JCS_RGB; /* Create a buffer for the write scan line which is where we will - * put the converted pixels (BRGx -> RGB) */ + * put the converted pixels (BGRx -> RGB) */ int write_stride = cinfo.image_width * cinfo.input_components; unsigned char *scanline_data = malloc(write_stride); memset(scanline_data, 0, write_stride); From 379c4462ca36ba1175c62bb1c6dd022d4964e614 Mon Sep 17 00:00:00 2001 From: Frode Langelo Date: Mon, 10 Aug 2015 09:12:29 -0700 Subject: [PATCH 11/27] GUAC-240: Implement adaptive encoding. Build a heat map of the screen, and use lossy image compression for areas refreshing frequently. Once refresh frequency is reduced the lossy area is repainted with a lossless image. --- src/common/Makefile.am | 46 +- src/common/guac_surface.c | 599 ++++++++++++++++++++++++--- src/common/guac_surface.h | 99 ++++- src/common/guac_surface_smoothness.c | 158 +++++++ src/common/guac_surface_smoothness.h | 43 ++ 5 files changed, 864 insertions(+), 81 deletions(-) create mode 100644 src/common/guac_surface_smoothness.c create mode 100644 src/common/guac_surface_smoothness.h diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 4f4b02c8..816e1a74 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -25,29 +25,31 @@ ACLOCAL_AMFLAGS = -I m4 noinst_LTLIBRARIES = libguac_common.la -noinst_HEADERS = \ - guac_io.h \ - guac_clipboard.h \ - guac_dot_cursor.h \ - guac_iconv.h \ - guac_json.h \ - guac_list.h \ - guac_pointer_cursor.h \ - guac_rect.h \ - guac_string.h \ - guac_surface.h +noinst_HEADERS = \ + guac_io.h \ + guac_clipboard.h \ + guac_dot_cursor.h \ + guac_iconv.h \ + guac_json.h \ + guac_list.h \ + guac_pointer_cursor.h \ + guac_rect.h \ + guac_string.h \ + guac_surface.h \ + guac_surface_smoothness.h -libguac_common_la_SOURCES = \ - guac_io.c \ - guac_clipboard.c \ - guac_dot_cursor.c \ - guac_iconv.c \ - guac_json.c \ - guac_list.c \ - guac_pointer_cursor.c \ - guac_rect.c \ - guac_string.c \ - guac_surface.c +libguac_common_la_SOURCES = \ + guac_io.c \ + guac_clipboard.c \ + guac_dot_cursor.c \ + guac_iconv.c \ + guac_json.c \ + guac_list.c \ + guac_pointer_cursor.c \ + guac_rect.c \ + guac_string.c \ + guac_surface.c \ + guac_surface_smoothness.c libguac_common_la_CFLAGS = \ -Werror -Wall -pedantic \ diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 8780c052..3c1e4d74 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -23,15 +23,18 @@ #include "config.h" #include "guac_rect.h" #include "guac_surface.h" +#include "guac_surface_smoothness.h" #include #include #include #include #include +#include #include #include +#include /** * The width of an update which should be considered negible and thus @@ -77,6 +80,425 @@ #define cairo_format_stride_for_width(format, width) (width*4) #endif +/** + * The JPEG compression minimum block size. This defines the optimal rectangle + * block size factor for JPEG compression to reduce artifacts. Usually this is + * 8 (8x8), but use 16 to reduce the occurence of ringing artifacts further. + */ +#define GUAC_SURFACE_JPEG_BLOCK_SIZE 16 + +/** + * Minimum JPEG bitmap size (area). If the bitmap is smaller than this + * threshold, it should be compressed as a PNG image to avoid the JPEG + * compression tax. + */ +#define GUAC_SURFACE_JPEG_MIN_BITMAP_SIZE 4096 + +/** + * The JPEG image quality ('quantization') setting to use. Range 0-100 where + * 100 is the highest quality/largest file size, and 0 is the lowest + * quality/smallest file size. + */ +#define GUAC_SURFACE_JPEG_IMAGE_QUALITY 90 + +/** + * Time (msec) between each time the surface's heat map is recalculated. + */ +#define GUAC_COMMON_SURFACE_HEAT_MAP_UPDATE_FREQ 2000 + +/** + * Refresh frequency threshold for when an area should be refreshed lossy. + */ +#define GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY 3 + +/** + * Time delay threshold between two updates where a lossy area will be moved + * to the non-lossy refresh pipe. + */ +#define GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD 3000 + +/* + * Forward declarations. + */ +static void __guac_common_clip_rect(guac_common_surface* surface, + guac_common_rect* rect, int* sx, int* sy); +static int __guac_common_should_combine(guac_common_surface* surface, + const guac_common_rect* rect, int rect_only); +static void __guac_common_mark_dirty(guac_common_surface* surface, + const guac_common_rect* rect); +static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface, + const guac_common_rect* rect); + +/** + * Flush a surface's lossy area to the dirty rectangle. This will make the + * rectangle refresh through the normal non-lossy refresh path. + * + * @param surface + * The surface whose lossy area will be moved to the dirty refresh + * queue. + * + * @param x + * The x coordinate of the area to move. + * + * @param y + * The y coordinate of the area to move. + */ +static void __guac_common_surface_flush_lossy_rect_to_dirty_rect( + guac_common_surface* surface, int x, int y) { + + /* Get the heat map index. */ + int hx = x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int hy = y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + + /* Don't update if this rect was not previously sent as a lossy refresh. */ + if (!surface->lossy_rect[hy][hx]) { + return; + } + + /* Clear the lossy status for this heat map rectangle. */ + surface->lossy_rect[hy][hx] = 0; + + guac_common_rect lossy_rect; + guac_common_rect_init(&lossy_rect, x, y, + GUAC_COMMON_SURFACE_HEAT_MAP_CELL, GUAC_COMMON_SURFACE_HEAT_MAP_CELL); + int sx = 0; + int sy = 0; + + /* Clip operation */ + __guac_common_clip_rect(surface, &lossy_rect, &sx, &sy); + if (lossy_rect.width <= 0 || lossy_rect.height <= 0) + return; + + /* Flush the rectangle if not combining. */ + if (!__guac_common_should_combine(surface, &lossy_rect, 0)) + guac_common_surface_flush_deferred(surface); + + /* Always defer draws */ + __guac_common_mark_dirty(surface, &lossy_rect); + +} + +/** + * Actual method which flushes a bitmap described by the dirty rectangle + * on the socket associated with the surface. + * + * The bitmap will be sent as a "jpeg" or "png" instruction based on the lossy + * flag. Certain conditions may override the lossy flag and send a lossless + * update. + * + * @param surface + * The surface whose dirty area will be flushed. + * + * @param dirty_rect + * The dirty rectangle. + * + * @param lossy + * Flag indicating whether this refresh should be lossy. + */ +static void __guac_common_surface_flush_to_bitmap_impl(guac_common_surface* surface, + guac_common_rect* dirty_rect, int lossy) { + + guac_socket* socket = surface->socket; + const guac_layer* layer = surface->layer; + int send_jpeg = 0; + + /* Set the JPEG flag indicating whether this bitmap should be sent as JPEG. + * Only send as a JPEG if the dirty is larger than the minimum JPEG bitmap + * size to avoid the JPEG image compression tax. */ + if (lossy && + (dirty_rect->width * dirty_rect->height) > GUAC_SURFACE_JPEG_MIN_BITMAP_SIZE) { + + /* Check the smoothness of the dirty rectangle. If smooth, do not send + * a JPEG as it has a higher overhead than standard PNG. */ + if (!guac_common_surface_rect_is_smooth(surface, dirty_rect)) { + + send_jpeg = 1; + + /* Tweak the rectangle if it is to be sent as JPEG so the size + * matches the JPEG block size. */ + guac_common_rect max; + guac_common_rect_init(&max, 0, 0, surface->width, surface->height); + + guac_common_rect_expand_to_grid(GUAC_SURFACE_JPEG_BLOCK_SIZE, + dirty_rect, &max); + } + + } + + /* Get Cairo surface for specified rect. + * The buffer is created with 4 bytes per pixel because Cairo's 24 bit RGB + * really is 32 bit BGRx */ + unsigned char* buffer = surface->buffer + dirty_rect->y * surface->stride + dirty_rect->x * 4; + cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24, + dirty_rect->width, + dirty_rect->height, + surface->stride); + + /* Send bitmap update for the dirty rectangle */ + if (send_jpeg) { + guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer, + dirty_rect->x, dirty_rect->y, rect, + GUAC_SURFACE_JPEG_IMAGE_QUALITY); + } + else { + guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, layer, + dirty_rect->x, dirty_rect->y, rect); + } + + cairo_surface_destroy(rect); + +} + +/** + * Flushes the bitmap update currently described by a lossy rectangle within the + * given surface. + * + * Scans through the regular bitmap update queue and excludes any rectangles + * covered by the lossy rectangle. + * + * @param surface + * The surface whose lossy area will be flushed. + */ +static void __guac_common_surface_flush_lossy_bitmap( + guac_common_surface* surface) { + + if (surface->lossy_dirty) { + + guac_common_surface_bitmap_rect* current = surface->bitmap_queue; + int original_queue_length = surface->bitmap_queue_length; + + /* Identify all bitmaps in queue which are + * covered by the lossy rectangle. */ + for (int i=0; i < original_queue_length; i++) { + + int intersects = guac_common_rect_intersects(¤t->rect, + &surface->lossy_dirty_rect); + /* Complete intersection. */ + if (intersects == 2) { + + /* Exclude this from the normal refresh as it is completely + * covered by the lossy dirty rectangle. */ + current->flushed = 1; + + } + + /* Partial intersection. + * The rectangle will be split if there is room on the queue. */ + else if (intersects == 1 && + surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE-5) { + + /* Clip and split rectangle into rectangles that are outside the + * lossy rectangle which are added to the normal refresh queue. + * The remaining rectangle which overlaps with the lossy + * rectangle is marked flushed to not be refreshed in the normal + * refresh cycle. + */ + guac_common_rect split_rect; + while (guac_common_rect_clip_and_split(¤t->rect, + &surface->lossy_dirty_rect, &split_rect)) { + + /* Add new rectangle to update queue */ + __guac_common_surface_flush_rect_to_queue(surface, + &split_rect); + + } + + /* Exclude the remaining part of the dirty rectangle + * which is completely covered by the lossy dirty rectangle. */ + current->flushed = 1; + + } + current++; + + } + + /* Flush the lossy bitmap */ + __guac_common_surface_flush_to_bitmap_impl(surface, + &surface->lossy_dirty_rect, 1); + + /* Flag this area as lossy so it can be moved back to the + * dirty rect and refreshed normally when refreshed less frequently. */ + int x = surface->lossy_dirty_rect.x; + int y = surface->lossy_dirty_rect.y; + int w = (x + surface->lossy_dirty_rect.width) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int h = (y + surface->lossy_dirty_rect.height) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + x /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + y /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + + for (int j = y; j <= h; j++) { + for (int i = x; i <= w; i++) { + surface->lossy_rect[j][i] = 1; + } + } + + /* Clear the lossy dirty flag. */ + surface->lossy_dirty = 0; + } + +} + +/** + * Calculate the current average refresh frequency for a given area on the + * surface. + * + * @param surface + * The surface on which the refresh frequency will be calculated. + * + * @param x + * The x coordinate for the area. + * + * @param y + * The y coordinate for the area. + * + * @param w + * The area width. + * + * @param h + * The area height. + * + * @return + * The average refresh frequency. + */ +static unsigned int __guac_common_surface_calculate_refresh_frequency( + guac_common_surface* surface, + int x, int y, int w, int h) +{ + + w = (x + w) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + h = (y + h) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + x /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + y /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + + unsigned int sum_frequency = 0; + unsigned int count = 0; + /* Iterate over all the heat map cells for the area + * and calculate the average refresh frequency. */ + for (int hy = y; hy <= h; hy++) { + for (int hx = x; hx <= w; hx++) { + + const guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx]; + sum_frequency += heat_rect->frequency; + count++; + + } + } + + /* Calculate the average. */ + if (count) { + return sum_frequency / count; + } + else { + return 0; + } +} + +/** + * Update the heat map for the surface and re-calculate the refresh frequencies. + * + * Any areas of the surface which have not been updated within a given threshold + * will be moved from the lossy to the normal refresh path. + * + * @param surface + * The surface on which the heat map will be refreshed. + * + * @param now + * The current time. + */ +static void __guac_common_surface_update_heat_map(guac_common_surface* surface, + guac_timestamp now) +{ + + /* Only update the heat map at the given interval. */ + if (now - surface->last_heat_map_update < GUAC_COMMON_SURFACE_HEAT_MAP_UPDATE_FREQ) { + return; + } + surface->last_heat_map_update = now; + + const int width = surface->width / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + const int height = surface->height / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int hx, hy; + + for (hy = 0; hy < height; hy++) { + for (hx = 0; hx < width; hx++) { + + guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx]; + + const int last_update_index = (heat_rect->index + GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ - 1) % GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ; + const guac_timestamp last_update = heat_rect->updates[last_update_index]; + const guac_timestamp time_since_last = now - last_update; + + /* If the time between the last 2 refreshes is larger than the + * threshold, move this rectangle back to the non-lossy + * refresh pipe. */ + if (time_since_last > GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD) { + + /* Send this lossy rectangle to the normal update queue. */ + const int x = hx * GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + const int y = hy * GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + __guac_common_surface_flush_lossy_rect_to_dirty_rect(surface, + x, y); + + /* Clear the frequency and refresh times for this square. */ + heat_rect->frequency = 0; + memset(heat_rect->updates, 0, sizeof(heat_rect->updates)); + continue ; + } + + /* Only calculate frequency after N updates to this heat + * rectangle. */ + if (heat_rect->updates[GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ - 1] == 0) { + continue; + } + + /* Calculate refresh frequency. */ + const guac_timestamp first_update = heat_rect->updates[heat_rect->index]; + int elapsed_time = last_update - first_update; + if (elapsed_time) + heat_rect->frequency = GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ * 1000 / elapsed_time; + else + heat_rect->frequency = 0; + + } + } + +} + +/** + * Touch the heat map with this update rectangle, so that the update + * frequency can be calculated later. + * + * @param surface + * The surface containing the rectangle to be updated. + * + * @param rect + * The rectangle updated. + * + * @param time + * The time stamp of this update. + */ +static void __guac_common_surface_touch_rect(guac_common_surface* surface, + guac_common_rect* rect, guac_timestamp time) +{ + + const int w = (rect->x + rect->width) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + const int h = (rect->y + rect->height) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int hx = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int hy = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + + for (; hy <= h; hy++) { + for (; hx <= w; hx++) { + + guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx]; + heat_rect->updates[heat_rect->index] = time; + + /* Move the heat index to the next. */ + heat_rect->index = (heat_rect->index + 1) % GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ; + + } + } + +} + /** * Updates the coordinates of the given rectangle to be within the bounds of * the given surface. @@ -190,7 +612,7 @@ static int __guac_common_should_combine(guac_common_surface* surface, const guac } } - + /* Otherwise, do not combine */ return 0; @@ -222,28 +644,69 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co } /** - * Flushes the PNG update currently described by the dirty rectangle within the - * given surface to that surface's PNG queue. There MUST be space within the + * Expands the lossy dirty rectangle of the given surface to contain the + * rectangle described by the given coordinates. + * + * @param surface + * The surface to mark as dirty. + * + * @param rect + * The rectangle of the update which is dirtying the surface. + */ +static void __guac_common_mark_lossy_dirty(guac_common_surface* surface, + const guac_common_rect* rect) { + + /* Ignore empty rects */ + if (rect->width <= 0 || rect->height <= 0) + return; + + /* If already dirty, update existing rect */ + if (surface->lossy_dirty) { + guac_common_rect_extend(&surface->lossy_dirty_rect, rect); + } + /* Otherwise init lossy dirty rect */ + else { + surface->lossy_dirty_rect = *rect; + surface->lossy_dirty = 1; + } + +} + +/** + * Flushes the rectangle to the given surface's bitmap queue. There MUST be + * space within the queue. + * + * @param surface The surface queue to flush to. + * @param rect The rectangle to flush. + */ +static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface, + const guac_common_rect* rect) { + guac_common_surface_bitmap_rect* bitmap_rect; + + /* Add new rect to queue */ + bitmap_rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]); + bitmap_rect->rect = *rect; + bitmap_rect->flushed = 0; +} + +/** + * Flushes the bitmap update currently described by the dirty rectangle within the + * given surface to that surface's bitmap queue. There MUST be space within the * queue. * * @param surface The surface to flush. */ static void __guac_common_surface_flush_to_queue(guac_common_surface* surface) { - guac_common_surface_png_rect* rect; - /* Do not flush if not dirty */ if (!surface->dirty) return; /* Add new rect to queue */ - rect = &(surface->png_queue[surface->png_queue_length++]); - rect->rect = surface->dirty_rect; - rect->flushed = 0; + __guac_common_surface_flush_rect_to_queue(surface, &surface->dirty_rect); /* Surface now flushed */ surface->dirty = 0; - } void guac_common_surface_flush_deferred(guac_common_surface* surface) { @@ -254,7 +717,7 @@ void guac_common_surface_flush_deferred(guac_common_surface* surface) { /* Flush if queue size has reached maximum (space is reserved for the final dirty rect, * as guac_common_surface_flush() MAY add an additional rect to the queue */ - if (surface->png_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1) + if (surface->bitmap_queue_length == GUAC_COMMON_SURFACE_QUEUE_SIZE-1) guac_common_surface_flush(surface); /* Append dirty rect to queue */ @@ -672,7 +1135,7 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, surface->width = w; surface->height = h; surface->dirty = 0; - surface->png_queue_length = 0; + surface->bitmap_queue_length = 0; /* Create corresponding Cairo surface */ surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w); @@ -691,6 +1154,22 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, else surface->realized = 0; + /* Initialize heat map and adaptive coding bits. */ + surface->lossy_dirty = 0; + surface->last_heat_map_update = 0; + for (int y = 0; y < GUAC_COMMON_SURFACE_HEAT_MAP_ROWS; y++) { + for (int x = 0; x < GUAC_COMMON_SURFACE_HEAT_MAP_COLS; x++) { + + guac_common_surface_heat_rect *rect= & surface->heat_map[y][x]; + memset(rect->updates, 0, sizeof(rect->updates)); + rect->frequency = 0; + rect->index = 0; + + surface->lossy_rect[y][x] = 0; + + } + } + return surface; } @@ -773,12 +1252,30 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_ if (rect.width <= 0 || rect.height <= 0) return; - /* Flush if not combining */ - if (!__guac_common_should_combine(surface, &rect, 0)) - guac_common_surface_flush_deferred(surface); + unsigned int freq = 0; - /* Always defer draws */ - __guac_common_mark_dirty(surface, &rect); + /* Update the heat map for the update rectangle. */ + guac_timestamp time = guac_timestamp_current(); + __guac_common_surface_touch_rect(surface, &rect, time); + + /* Calculate the update frequency for this rectangle. */ + freq = __guac_common_surface_calculate_refresh_frequency(surface, x, y, w, h); + + /* If this rectangle is hot, mark lossy dirty rectangle. */ + if (freq >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY) { + __guac_common_mark_lossy_dirty(surface, &rect); + } + /* Standard refresh path */ + else { + + /* Flush if not combining */ + if (!__guac_common_should_combine(surface, &rect, 0)) + guac_common_surface_flush_deferred(surface); + + /* Always defer draws */ + __guac_common_mark_dirty(surface, &rect); + + } } @@ -948,30 +1445,25 @@ void guac_common_surface_reset_clip(guac_common_surface* surface) { } /** - * Flushes the PNG update currently described by the dirty rectangle within the - * given surface directly to a "png" instruction, which is sent on the socket - * associated with the surface. + * Flushes the bitmap update currently described by the dirty rectangle within the + * given surface. * * @param surface The surface to flush. */ -static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { +static void __guac_common_surface_flush_to_bitmap(guac_common_surface* surface) { if (surface->dirty) { - guac_socket* socket = surface->socket; - const guac_layer* layer = surface->layer; + guac_common_rect dirty_rect; + guac_common_rect_init(&dirty_rect, + surface->dirty_rect.x, + surface->dirty_rect.y, + surface->dirty_rect.width, + surface->dirty_rect.height); - /* Get Cairo surface for specified rect */ - unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4; - cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24, - surface->dirty_rect.width, - surface->dirty_rect.height, - surface->stride); + /* Flush bitmap */ + __guac_common_surface_flush_to_bitmap_impl(surface, &dirty_rect, 0); - /* Send PNG for rect */ - guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, - layer, surface->dirty_rect.x, surface->dirty_rect.y, rect); - cairo_surface_destroy(rect); surface->realized = 1; /* Surface is no longer dirty */ @@ -982,15 +1474,15 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { } /** - * Comparator for instances of guac_common_surface_png_rect, the elements - * which make up a surface's PNG buffer. + * Comparator for instances of guac_common_surface_bitmap_rect, the elements + * which make up a surface's bitmap buffer. * * @see qsort */ -static int __guac_common_surface_png_rect_compare(const void* a, const void* b) { +static int __guac_common_surface_bitmap_rect_compare(const void* a, const void* b) { - guac_common_surface_png_rect* ra = (guac_common_surface_png_rect*) a; - guac_common_surface_png_rect* rb = (guac_common_surface_png_rect*) b; + guac_common_surface_bitmap_rect* ra = (guac_common_surface_bitmap_rect*) a; + guac_common_surface_bitmap_rect* rb = (guac_common_surface_bitmap_rect*) b; /* Order roughly top to bottom, left to right */ if (ra->rect.y != rb->rect.y) return ra->rect.y - rb->rect.y; @@ -1006,31 +1498,38 @@ static int __guac_common_surface_png_rect_compare(const void* a, const void* b) void guac_common_surface_flush(guac_common_surface* surface) { - guac_common_surface_png_rect* current = surface->png_queue; + /* Update heat map. */ + guac_timestamp time = guac_timestamp_current(); + __guac_common_surface_update_heat_map(surface, time); + /* Flush final dirty rectangle to queue. */ + __guac_common_surface_flush_to_queue(surface); + + /* Flush the lossy bitmap to client. */ + __guac_common_surface_flush_lossy_bitmap(surface); + + guac_common_surface_bitmap_rect* current = surface->bitmap_queue; int i, j; int original_queue_length; int flushed = 0; - /* Flush final dirty rect to queue */ - __guac_common_surface_flush_to_queue(surface); - original_queue_length = surface->png_queue_length; + original_queue_length = surface->bitmap_queue_length; /* Sort updates to make combination less costly */ - qsort(surface->png_queue, surface->png_queue_length, sizeof(guac_common_surface_png_rect), - __guac_common_surface_png_rect_compare); + qsort(surface->bitmap_queue, surface->bitmap_queue_length, sizeof(guac_common_surface_bitmap_rect), + __guac_common_surface_bitmap_rect_compare); /* Flush all rects in queue */ - for (i=0; i < surface->png_queue_length; i++) { + for (i=0; i < surface->bitmap_queue_length; i++) { /* Get next unflushed candidate */ - guac_common_surface_png_rect* candidate = current; + guac_common_surface_bitmap_rect* candidate = current; if (!candidate->flushed) { int combined = 0; /* Build up rect as much as possible */ - for (j=i; j < surface->png_queue_length; j++) { + for (j=i; j < surface->bitmap_queue_length; j++) { if (!candidate->flushed) { @@ -1054,13 +1553,13 @@ void guac_common_surface_flush(guac_common_surface* surface) { /* Re-add to queue if there's room and this update was modified or we expect others might be */ if ((combined > 1 || i < original_queue_length) - && surface->png_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE) + && surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE) __guac_common_surface_flush_to_queue(surface); - /* Flush as PNG otherwise */ + /* Flush as bitmap otherwise */ else { if (surface->dirty) flushed++; - __guac_common_surface_flush_to_png(surface); + __guac_common_surface_flush_to_bitmap(surface); } } @@ -1070,7 +1569,7 @@ void guac_common_surface_flush(guac_common_surface* surface) { } /* Flush complete */ - surface->png_queue_length = 0; + surface->bitmap_queue_length = 0; } diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index cacfc2d9..90304fd2 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -33,15 +33,69 @@ #include /** - * The maximum number of updates to allow within the PNG queue. + * The maximum number of updates to allow within the bitmap queue. */ #define GUAC_COMMON_SURFACE_QUEUE_SIZE 256 /** - * Representation of a PNG update, having a rectangle of image data (stored + * The maximum surface width; 2x WQXGA @ 16:10. + */ +#define GUAC_COMMON_SURFACE_MAX_WIDTH 5120 + +/** + * The maximum surface height; 2x WQXGA @ 16:10. + */ +#define GUAC_COMMON_SURFACE_MAX_HEIGHT 3200 + +/** + * Heat map square size in pixels. + */ +#define GUAC_COMMON_SURFACE_HEAT_MAP_CELL 64 + +/** + * Heat map number of columns. + */ +#define GUAC_COMMON_SURFACE_HEAT_MAP_COLS (GUAC_COMMON_SURFACE_MAX_WIDTH / GUAC_COMMON_SURFACE_HEAT_MAP_CELL) + +/** + * Heat map number of rows. + */ +#define GUAC_COMMON_SURFACE_HEAT_MAP_ROWS (GUAC_COMMON_SURFACE_MAX_HEIGHT / GUAC_COMMON_SURFACE_HEAT_MAP_CELL) + +/** + * The number of time stamps to collect to be able to calculate the refresh + * frequency for a heat map cell. + */ +#define GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ 5 + +/** + * Representation of a rectangle or cell in the refresh heat map. This rectangle + * is used to keep track of how often an area on a surface is refreshed. + */ +typedef struct guac_common_surface_heat_rect { + + /** + * Time of the last N updates, used to calculate the refresh frequency. + */ + guac_timestamp updates[GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ]; + + /** + * Index of the next update slot in the updates array. + */ + int index; + + /** + * The current update frequency. + */ + unsigned int frequency; + +} guac_common_surface_heat_rect; + +/** + * Representation of a bitmap update, having a rectangle of image data (stored * elsewhere) and a flushed/not-flushed state. */ -typedef struct guac_common_surface_png_rect { +typedef struct guac_common_surface_bitmap_rect { /** * Whether this rectangle has been flushed. @@ -49,11 +103,11 @@ typedef struct guac_common_surface_png_rect { int flushed; /** - * The rectangle containing the PNG update. + * The rectangle containing the bitmap update. */ guac_common_rect rect; -} guac_common_surface_png_rect; +} guac_common_surface_bitmap_rect; /** * Surface which backs a Guacamole buffer or layer, automatically @@ -123,14 +177,41 @@ typedef struct guac_common_surface { guac_common_rect clip_rect; /** - * The number of updates in the PNG queue. + * The number of updates in the bitmap queue. */ - int png_queue_length; + int bitmap_queue_length; /** - * All queued PNG updates. + * All queued bitmap updates. */ - guac_common_surface_png_rect png_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE]; + guac_common_surface_bitmap_rect bitmap_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE]; + + /** + * Last time the heat map was refreshed. + */ + guac_timestamp last_heat_map_update; + + /** + * A heat map keeping track of the refresh frequency of + * the areas of the screen. + */ + guac_common_surface_heat_rect heat_map[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS]; + + /* + * Map of areas currently refreshed lossy. + */ + int lossy_rect[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS]; + + /** + * Non-zero if this surface's lossy area is dirty and needs to be flushed, + * 0 otherwise. + */ + int lossy_dirty; + + /** + * The lossy area's dirty rectangle. + */ + guac_common_rect lossy_dirty_rect; } guac_common_surface; diff --git a/src/common/guac_surface_smoothness.c b/src/common/guac_surface_smoothness.c new file mode 100644 index 00000000..9300573b --- /dev/null +++ b/src/common/guac_surface_smoothness.c @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2015 Glyptodon LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/* + * Smoothness detection from: + * QEMU VNC display driver: tight encoding + * + * From libvncserver/libvncserver/tight.c + * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved. + * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. + * + * Copyright (C) 2010 Corentin Chary + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "guac_surface_smoothness.h" + +#include +#include +#include + +/** + * The threshold to determine an image to be smooth. + */ +#define GUAC_SURFACE_SMOOTHNESS_THRESHOLD 0 + +/** + * Width of sub-row when detecting image smoothness. + */ +#define GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH 7 + +int guac_common_surface_rect_is_smooth(guac_common_surface* surface, + guac_common_rect* rect) +{ + + /* + * Code to guess if the image in a given rectangle is smooth + * (by applying "gradient" filter or JPEG coder). + */ + int x, y, d, dx; + unsigned int c; + unsigned int stats[256]; + int pixels = 0; + int pix, left[3]; + unsigned char* buffer = surface->buffer; + int stride = surface->stride; + int w = rect->x + rect->width; + int h = rect->y + rect->height; + + /* If rect is out of bounds, bail out */ + if (rect->x < 0 || rect->y < 0 || + w > surface->width || h > surface->height) { + return 0; + } + + /* If rect is too small to process, bail out */ + if (rect->width < GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH + 1 || + rect->height < GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH + 1) { + return 0; + } + + /* Init stats array */ + memset(stats, 0, sizeof (stats)); + + for (y = rect->y, x = rect->x; y < h && x < w;) { + + /* Scan sub-sections of the surface to determine how close the colors are + * to the previous. */ + for (d = 0; + d < h - y && d < w - x - GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH; + d++) { + + for (c = 0; c < 3; c++) { + unsigned int index = (y+d)*stride + (x+d)*4 + c; + left[c] = buffer[index] & 0xFF; + } + + for (dx = 1; dx <= GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH; dx++) { + + for (c = 0; c < 3; c++) { + unsigned int index = (y+d)*stride + (x+d+dx)*4 + c; + pix = buffer[index] & 0xFF; + stats[abs(pix - left[c])]++; + left[c] = pix; + } + ++pixels; + } + } + + /* Advance to next section */ + if (w > h) { + x += h; + y = rect->y; + } else { + x = rect->x; + y += w; + } + } + + if (pixels == 0) { + return 1; + } + + /* 95% smooth or more */ + if (stats[0] * 33 / pixels >= 95) { + return 1; + } + + unsigned int smoothness = 0; + for (c = 1; c < 8; c++) { + smoothness += stats[c] * (c * c); + if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { + return 1; + } + } + for (; c < 256; c++) { + smoothness += stats[c] * (c * c); + } + smoothness /= (pixels * 3 - stats[0]); + + return smoothness <= GUAC_SURFACE_SMOOTHNESS_THRESHOLD; +} diff --git a/src/common/guac_surface_smoothness.h b/src/common/guac_surface_smoothness.h new file mode 100644 index 00000000..7bd15145 --- /dev/null +++ b/src/common/guac_surface_smoothness.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2015 Glyptodon LLC + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef __GUAC_COMMON_SURFACE_SMOOTHNESS_H +#define __GUAC_COMMON_SURFACE_SMOOTHNESS_H + +#include "guac_surface.h" + +/** + * Returns the smoothness of an area on a surface. + * + * @param surface + * The surface on which the rectangle exists. + * + * @param rect + * The rectangle to check for smoothness. + * + * @return + * 1 if rectangle is smooth, zero if not. + */ +int guac_common_surface_rect_is_smooth(guac_common_surface* surface, + guac_common_rect* rect); + +#endif From 254a0dded0f4d8034800d86db22771dd46407916 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Wed, 12 Aug 2015 21:29:37 -0700 Subject: [PATCH 12/27] GUAC-240: Remove need for forward declarations of static functions. --- src/common/guac_surface.c | 328 ++++++++++++++++++-------------------- 1 file changed, 158 insertions(+), 170 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 3c1e4d74..219cfcf1 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -117,17 +117,149 @@ */ #define GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD 3000 -/* - * Forward declarations. +/** + * Updates the coordinates of the given rectangle to be within the bounds of + * the given surface. + * + * @param surface The surface to use for clipping. + * @param rect The rectangle to clip. + * @param sx The X coordinate of the source rectangle, if any. + * @param sy The Y coordinate of the source rectangle, if any. + */ +static void __guac_common_bound_rect(guac_common_surface* surface, + guac_common_rect* rect, int* sx, int* sy) { + + guac_common_rect bounds_rect = { + .x = 0, + .y = 0, + .width = surface->width, + .height = surface->height + }; + + int orig_x = rect->x; + int orig_y = rect->y; + + guac_common_rect_constrain(rect, &bounds_rect); + + /* Update source X/Y if given */ + if (sx != NULL) *sx += rect->x - orig_x; + if (sy != NULL) *sy += rect->y - orig_y; + +} + +/** + * Updates the coordinates of the given rectangle to be within the clipping + * rectangle of the given surface, which must always be within the bounding + * rectangle of the given surface. + * + * @param surface The surface to use for clipping. + * @param rect The rectangle to clip. + * @param sx The X coordinate of the source rectangle, if any. + * @param sy The Y coordinate of the source rectangle, if any. */ static void __guac_common_clip_rect(guac_common_surface* surface, - guac_common_rect* rect, int* sx, int* sy); -static int __guac_common_should_combine(guac_common_surface* surface, - const guac_common_rect* rect, int rect_only); -static void __guac_common_mark_dirty(guac_common_surface* surface, - const guac_common_rect* rect); -static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface, - const guac_common_rect* rect); + guac_common_rect* rect, int* sx, int* sy) { + + int orig_x = rect->x; + int orig_y = rect->y; + + /* Just bound within surface if no clipping rectangle applied */ + if (!surface->clipped) { + __guac_common_bound_rect(surface, rect, sx, sy); + return; + } + + guac_common_rect_constrain(rect, &surface->clip_rect); + + /* Update source X/Y if given */ + if (sx != NULL) *sx += rect->x - orig_x; + if (sy != NULL) *sy += rect->y - orig_y; + +} + +/** + * Returns whether the given rectangle should be combined into the existing + * dirty rectangle, to be eventually flushed as a "png" instruction. + * + * @param surface The surface to be queried. + * @param rect The update rectangle. + * @param rect_only Non-zero if this update, by its nature, contains only + * metainformation about the update's rectangle, zero if + * the update also contains image data. + * @return Non-zero if the update should be combined with any existing update, + * zero otherwise. + */ +static int __guac_common_should_combine(guac_common_surface* surface, const guac_common_rect* rect, int rect_only) { + + if (surface->dirty) { + + int combined_cost, dirty_cost, update_cost; + + /* Simulate combination */ + guac_common_rect combined = surface->dirty_rect; + guac_common_rect_extend(&combined, rect); + + /* Combine if result is still small */ + if (combined.width <= GUAC_SURFACE_NEGLIGIBLE_WIDTH && combined.height <= GUAC_SURFACE_NEGLIGIBLE_HEIGHT) + return 1; + + /* Estimate costs of the existing update, new update, and both combined */ + combined_cost = GUAC_SURFACE_BASE_COST + combined.width * combined.height; + dirty_cost = GUAC_SURFACE_BASE_COST + surface->dirty_rect.width * surface->dirty_rect.height; + update_cost = GUAC_SURFACE_BASE_COST + rect->width * rect->height; + + /* Reduce cost if no image data */ + if (rect_only) + update_cost /= GUAC_SURFACE_DATA_FACTOR; + + /* Combine if cost estimate shows benefit */ + if (combined_cost <= update_cost + dirty_cost) + return 1; + + /* Combine if increase in cost is negligible */ + if (combined_cost - dirty_cost <= dirty_cost / GUAC_SURFACE_NEGLIGIBLE_INCREASE) + return 1; + + if (combined_cost - update_cost <= update_cost / GUAC_SURFACE_NEGLIGIBLE_INCREASE) + return 1; + + /* Combine if we anticipate further updates, as this update follows a common fill pattern */ + if (rect->x == surface->dirty_rect.x && rect->y == surface->dirty_rect.y + surface->dirty_rect.height) { + if (combined_cost <= (dirty_cost + update_cost) * GUAC_SURFACE_FILL_PATTERN_FACTOR) + return 1; + } + + } + + /* Otherwise, do not combine */ + return 0; + +} + +/** + * Expands the dirty rect of the given surface to contain the rect described by the given + * coordinates. + * + * @param surface The surface to mark as dirty. + * @param rect The rectangle of the update which is dirtying the surface. + */ +static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_common_rect* rect) { + + /* Ignore empty rects */ + if (rect->width <= 0 || rect->height <= 0) + return; + + /* If already dirty, update existing rect */ + if (surface->dirty) + guac_common_rect_extend(&surface->dirty_rect, rect); + + /* Otherwise init dirty rect */ + else { + surface->dirty_rect = *rect; + surface->dirty = 1; + } + +} /** * Flush a surface's lossy area to the dirty rectangle. This will make the @@ -249,6 +381,23 @@ static void __guac_common_surface_flush_to_bitmap_impl(guac_common_surface* surf } +/** + * Flushes the rectangle to the given surface's bitmap queue. There MUST be + * space within the queue. + * + * @param surface The surface queue to flush to. + * @param rect The rectangle to flush. + */ +static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface, + const guac_common_rect* rect) { + guac_common_surface_bitmap_rect* bitmap_rect; + + /* Add new rect to queue */ + bitmap_rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]); + bitmap_rect->rect = *rect; + bitmap_rect->flushed = 0; +} + /** * Flushes the bitmap update currently described by a lossy rectangle within the * given surface. @@ -499,150 +648,6 @@ static void __guac_common_surface_touch_rect(guac_common_surface* surface, } -/** - * Updates the coordinates of the given rectangle to be within the bounds of - * the given surface. - * - * @param surface The surface to use for clipping. - * @param rect The rectangle to clip. - * @param sx The X coordinate of the source rectangle, if any. - * @param sy The Y coordinate of the source rectangle, if any. - */ -static void __guac_common_bound_rect(guac_common_surface* surface, - guac_common_rect* rect, int* sx, int* sy) { - - guac_common_rect bounds_rect = { - .x = 0, - .y = 0, - .width = surface->width, - .height = surface->height - }; - - int orig_x = rect->x; - int orig_y = rect->y; - - guac_common_rect_constrain(rect, &bounds_rect); - - /* Update source X/Y if given */ - if (sx != NULL) *sx += rect->x - orig_x; - if (sy != NULL) *sy += rect->y - orig_y; - -} - -/** - * Updates the coordinates of the given rectangle to be within the clipping - * rectangle of the given surface, which must always be within the bounding - * rectangle of the given surface. - * - * @param surface The surface to use for clipping. - * @param rect The rectangle to clip. - * @param sx The X coordinate of the source rectangle, if any. - * @param sy The Y coordinate of the source rectangle, if any. - */ -static void __guac_common_clip_rect(guac_common_surface* surface, - guac_common_rect* rect, int* sx, int* sy) { - - int orig_x = rect->x; - int orig_y = rect->y; - - /* Just bound within surface if no clipping rectangle applied */ - if (!surface->clipped) { - __guac_common_bound_rect(surface, rect, sx, sy); - return; - } - - guac_common_rect_constrain(rect, &surface->clip_rect); - - /* Update source X/Y if given */ - if (sx != NULL) *sx += rect->x - orig_x; - if (sy != NULL) *sy += rect->y - orig_y; - -} - -/** - * Returns whether the given rectangle should be combined into the existing - * dirty rectangle, to be eventually flushed as a "png" instruction. - * - * @param surface The surface to be queried. - * @param rect The update rectangle. - * @param rect_only Non-zero if this update, by its nature, contains only - * metainformation about the update's rectangle, zero if - * the update also contains image data. - * @return Non-zero if the update should be combined with any existing update, - * zero otherwise. - */ -static int __guac_common_should_combine(guac_common_surface* surface, const guac_common_rect* rect, int rect_only) { - - if (surface->dirty) { - - int combined_cost, dirty_cost, update_cost; - - /* Simulate combination */ - guac_common_rect combined = surface->dirty_rect; - guac_common_rect_extend(&combined, rect); - - /* Combine if result is still small */ - if (combined.width <= GUAC_SURFACE_NEGLIGIBLE_WIDTH && combined.height <= GUAC_SURFACE_NEGLIGIBLE_HEIGHT) - return 1; - - /* Estimate costs of the existing update, new update, and both combined */ - combined_cost = GUAC_SURFACE_BASE_COST + combined.width * combined.height; - dirty_cost = GUAC_SURFACE_BASE_COST + surface->dirty_rect.width * surface->dirty_rect.height; - update_cost = GUAC_SURFACE_BASE_COST + rect->width * rect->height; - - /* Reduce cost if no image data */ - if (rect_only) - update_cost /= GUAC_SURFACE_DATA_FACTOR; - - /* Combine if cost estimate shows benefit */ - if (combined_cost <= update_cost + dirty_cost) - return 1; - - /* Combine if increase in cost is negligible */ - if (combined_cost - dirty_cost <= dirty_cost / GUAC_SURFACE_NEGLIGIBLE_INCREASE) - return 1; - - if (combined_cost - update_cost <= update_cost / GUAC_SURFACE_NEGLIGIBLE_INCREASE) - return 1; - - /* Combine if we anticipate further updates, as this update follows a common fill pattern */ - if (rect->x == surface->dirty_rect.x && rect->y == surface->dirty_rect.y + surface->dirty_rect.height) { - if (combined_cost <= (dirty_cost + update_cost) * GUAC_SURFACE_FILL_PATTERN_FACTOR) - return 1; - } - - } - - /* Otherwise, do not combine */ - return 0; - -} - -/** - * Expands the dirty rect of the given surface to contain the rect described by the given - * coordinates. - * - * @param surface The surface to mark as dirty. - * @param rect The rectangle of the update which is dirtying the surface. - */ -static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_common_rect* rect) { - - /* Ignore empty rects */ - if (rect->width <= 0 || rect->height <= 0) - return; - - /* If already dirty, update existing rect */ - if (surface->dirty) - guac_common_rect_extend(&surface->dirty_rect, rect); - - /* Otherwise init dirty rect */ - else { - surface->dirty_rect = *rect; - surface->dirty = 1; - } - -} - /** * Expands the lossy dirty rectangle of the given surface to contain the * rectangle described by the given coordinates. @@ -672,23 +677,6 @@ static void __guac_common_mark_lossy_dirty(guac_common_surface* surface, } -/** - * Flushes the rectangle to the given surface's bitmap queue. There MUST be - * space within the queue. - * - * @param surface The surface queue to flush to. - * @param rect The rectangle to flush. - */ -static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface, - const guac_common_rect* rect) { - guac_common_surface_bitmap_rect* bitmap_rect; - - /* Add new rect to queue */ - bitmap_rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]); - bitmap_rect->rect = *rect; - bitmap_rect->flushed = 0; -} - /** * Flushes the bitmap update currently described by the dirty rectangle within the * given surface to that surface's bitmap queue. There MUST be space within the From 807e3a39a527000a6ab4b9b363cad54ebe06b76e Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 01:08:58 -0700 Subject: [PATCH 13/27] GUAC-240: Simplify and clarify update history tracking. Remove lossless refresh of previously-lossy updates for now. --- src/common/guac_surface.c | 242 ++++++++++---------------------------- src/common/guac_surface.h | 25 ++-- 2 files changed, 78 insertions(+), 189 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 219cfcf1..53e49011 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -261,55 +261,6 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co } -/** - * Flush a surface's lossy area to the dirty rectangle. This will make the - * rectangle refresh through the normal non-lossy refresh path. - * - * @param surface - * The surface whose lossy area will be moved to the dirty refresh - * queue. - * - * @param x - * The x coordinate of the area to move. - * - * @param y - * The y coordinate of the area to move. - */ -static void __guac_common_surface_flush_lossy_rect_to_dirty_rect( - guac_common_surface* surface, int x, int y) { - - /* Get the heat map index. */ - int hx = x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int hy = y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - - /* Don't update if this rect was not previously sent as a lossy refresh. */ - if (!surface->lossy_rect[hy][hx]) { - return; - } - - /* Clear the lossy status for this heat map rectangle. */ - surface->lossy_rect[hy][hx] = 0; - - guac_common_rect lossy_rect; - guac_common_rect_init(&lossy_rect, x, y, - GUAC_COMMON_SURFACE_HEAT_MAP_CELL, GUAC_COMMON_SURFACE_HEAT_MAP_CELL); - int sx = 0; - int sy = 0; - - /* Clip operation */ - __guac_common_clip_rect(surface, &lossy_rect, &sx, &sy); - if (lossy_rect.width <= 0 || lossy_rect.height <= 0) - return; - - /* Flush the rectangle if not combining. */ - if (!__guac_common_should_combine(surface, &lossy_rect, 0)) - guac_common_surface_flush_deferred(surface); - - /* Always defer draws */ - __guac_common_mark_dirty(surface, &lossy_rect); - -} - /** * Actual method which flushes a bitmap described by the dirty rectangle * on the socket associated with the surface. @@ -508,107 +459,55 @@ static void __guac_common_surface_flush_lossy_bitmap( * @return * The average refresh frequency. */ -static unsigned int __guac_common_surface_calculate_refresh_frequency( - guac_common_surface* surface, - int x, int y, int w, int h) -{ +static unsigned int __guac_common_surface_calculate_framerate( + guac_common_surface* surface, guac_common_rect* rect) { - w = (x + w) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - h = (y + h) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - x /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - y /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int x, y; - unsigned int sum_frequency = 0; + /* Calculate minimum X/Y coordinates intersecting given rect */ + int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + + /* Calculate maximum X/Y coordinates intersecting given rect */ + int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + + unsigned int sum_framerate = 0; unsigned int count = 0; - /* Iterate over all the heat map cells for the area - * and calculate the average refresh frequency. */ - for (int hy = y; hy <= h; hy++) { - for (int hx = x; hx <= w; hx++) { - const guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx]; - sum_frequency += heat_rect->frequency; + /* Iterate over all the heat map cells for the area + * and calculate the average framerate */ + for (y = min_y; y < max_y; y++) { + for (x = min_x; x < max_x; x++) { + + const guac_common_surface_heat_rect* heat_rect = + &surface->heat_map[y][x]; + + /* Calculate indicies for latest and oldest history entries */ + int oldest_entry = heat_rect->oldest_entry; + int latest_entry = oldest_entry - 1; + if (latest_entry < 0) + latest_entry = GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE; + + /* Calculate elapsed time covering entire history for this cell */ + int elapsed_time = heat_rect->history[latest_entry] + - heat_rect->history[oldest_entry]; + + /* Calculate and add framerate */ + if (elapsed_time) + sum_framerate += GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE + * 1000 / elapsed_time; + count++; } } - /* Calculate the average. */ - if (count) { - return sum_frequency / count; - } - else { - return 0; - } -} + /* Calculate the average framerate over entire rect */ + if (count) + return sum_framerate / count; -/** - * Update the heat map for the surface and re-calculate the refresh frequencies. - * - * Any areas of the surface which have not been updated within a given threshold - * will be moved from the lossy to the normal refresh path. - * - * @param surface - * The surface on which the heat map will be refreshed. - * - * @param now - * The current time. - */ -static void __guac_common_surface_update_heat_map(guac_common_surface* surface, - guac_timestamp now) -{ - - /* Only update the heat map at the given interval. */ - if (now - surface->last_heat_map_update < GUAC_COMMON_SURFACE_HEAT_MAP_UPDATE_FREQ) { - return; - } - surface->last_heat_map_update = now; - - const int width = surface->width / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - const int height = surface->height / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int hx, hy; - - for (hy = 0; hy < height; hy++) { - for (hx = 0; hx < width; hx++) { - - guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx]; - - const int last_update_index = (heat_rect->index + GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ - 1) % GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ; - const guac_timestamp last_update = heat_rect->updates[last_update_index]; - const guac_timestamp time_since_last = now - last_update; - - /* If the time between the last 2 refreshes is larger than the - * threshold, move this rectangle back to the non-lossy - * refresh pipe. */ - if (time_since_last > GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD) { - - /* Send this lossy rectangle to the normal update queue. */ - const int x = hx * GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - const int y = hy * GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - __guac_common_surface_flush_lossy_rect_to_dirty_rect(surface, - x, y); - - /* Clear the frequency and refresh times for this square. */ - heat_rect->frequency = 0; - memset(heat_rect->updates, 0, sizeof(heat_rect->updates)); - continue ; - } - - /* Only calculate frequency after N updates to this heat - * rectangle. */ - if (heat_rect->updates[GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ - 1] == 0) { - continue; - } - - /* Calculate refresh frequency. */ - const guac_timestamp first_update = heat_rect->updates[heat_rect->index]; - int elapsed_time = last_update - first_update; - if (elapsed_time) - heat_rect->frequency = GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ * 1000 / elapsed_time; - else - heat_rect->frequency = 0; - - } - } + return 0; } @@ -626,22 +525,33 @@ static void __guac_common_surface_update_heat_map(guac_common_surface* surface, * The time stamp of this update. */ static void __guac_common_surface_touch_rect(guac_common_surface* surface, - guac_common_rect* rect, guac_timestamp time) -{ + guac_common_rect* rect, guac_timestamp time) { - const int w = (rect->x + rect->width) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - const int h = (rect->y + rect->height) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int hx = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int hy = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int x, y; - for (; hy <= h; hy++) { - for (; hx <= w; hx++) { + /* Calculate minimum X/Y coordinates intersecting given rect */ + int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - guac_common_surface_heat_rect* heat_rect = &surface->heat_map[hy][hx]; - heat_rect->updates[heat_rect->index] = time; + /* Calculate maximum X/Y coordinates intersecting given rect */ + int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - /* Move the heat index to the next. */ - heat_rect->index = (heat_rect->index + 1) % GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ; + /* Update all heat map cells which intersect with rectangle */ + for (y = min_y; y <= max_y; y++) { + for (x = min_x; x <= max_x; x++) { + + /* Get heat map cell at current location */ + guac_common_surface_heat_rect* heat_rect = &surface->heat_map[y][x]; + + /* Replace oldest entry with new timestamp */ + heat_rect->history[heat_rect->oldest_entry] = time; + + /* Update to next oldest entry */ + heat_rect->oldest_entry++; + if (heat_rect->oldest_entry >= + GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE) + heat_rect->oldest_entry = 0; } } @@ -1116,14 +1026,12 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, guac_socket* socket, const guac_layer* layer, int w, int h) { /* Init surface */ - guac_common_surface* surface = malloc(sizeof(guac_common_surface)); + guac_common_surface* surface = calloc(1, sizeof(guac_common_surface)); surface->client = client; surface->socket = socket; surface->layer = layer; surface->width = w; surface->height = h; - surface->dirty = 0; - surface->bitmap_queue_length = 0; /* Create corresponding Cairo surface */ surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w); @@ -1142,22 +1050,6 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, else surface->realized = 0; - /* Initialize heat map and adaptive coding bits. */ - surface->lossy_dirty = 0; - surface->last_heat_map_update = 0; - for (int y = 0; y < GUAC_COMMON_SURFACE_HEAT_MAP_ROWS; y++) { - for (int x = 0; x < GUAC_COMMON_SURFACE_HEAT_MAP_COLS; x++) { - - guac_common_surface_heat_rect *rect= & surface->heat_map[y][x]; - memset(rect->updates, 0, sizeof(rect->updates)); - rect->frequency = 0; - rect->index = 0; - - surface->lossy_rect[y][x] = 0; - - } - } - return surface; } @@ -1246,8 +1138,8 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_ guac_timestamp time = guac_timestamp_current(); __guac_common_surface_touch_rect(surface, &rect, time); - /* Calculate the update frequency for this rectangle. */ - freq = __guac_common_surface_calculate_refresh_frequency(surface, x, y, w, h); + /* Calculate the average framerate for this rectangle. */ + freq = __guac_common_surface_calculate_framerate(surface, &rect); /* If this rectangle is hot, mark lossy dirty rectangle. */ if (freq >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY) { @@ -1486,10 +1378,6 @@ static int __guac_common_surface_bitmap_rect_compare(const void* a, const void* void guac_common_surface_flush(guac_common_surface* surface) { - /* Update heat map. */ - guac_timestamp time = guac_timestamp_current(); - __guac_common_surface_update_heat_map(surface, time); - /* Flush final dirty rectangle to queue. */ __guac_common_surface_flush_to_queue(surface); diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index 90304fd2..50a20e2e 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -63,10 +63,11 @@ #define GUAC_COMMON_SURFACE_HEAT_MAP_ROWS (GUAC_COMMON_SURFACE_MAX_HEIGHT / GUAC_COMMON_SURFACE_HEAT_MAP_CELL) /** - * The number of time stamps to collect to be able to calculate the refresh - * frequency for a heat map cell. + * The number of entries to collect within each heat map cell. Collected + * history entries are used to determine the framerate of the region associated + * with that cell. */ -#define GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ 5 +#define GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE 5 /** * Representation of a rectangle or cell in the refresh heat map. This rectangle @@ -75,19 +76,19 @@ typedef struct guac_common_surface_heat_rect { /** - * Time of the last N updates, used to calculate the refresh frequency. + * Timestamps of each of the last N updates covering the location + * associated with this heat map cell. This is used to calculate the + * framerate. This array is structured as a ring buffer containing history + * entries in chronologically-ascending order, starting at the entry + * pointed to by oldest_entry and proceeding through all other entries, + * wrapping around if the end of the array is reached. */ - guac_timestamp updates[GUAC_COMMON_SURFACE_HEAT_UPDATE_ARRAY_SZ]; + guac_timestamp history[GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE]; /** - * Index of the next update slot in the updates array. + * Index of the oldest entry within the history. */ - int index; - - /** - * The current update frequency. - */ - unsigned int frequency; + int oldest_entry; } guac_common_surface_heat_rect; From 26f9070d99830d227d69a11c29834d672e90baf7 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 01:27:09 -0700 Subject: [PATCH 14/27] GUAC-240: Restore flush to PNG. Simplify handling of flush. Remove lossy pipeline. --- src/common/guac_surface.c | 272 +++++--------------------------------- src/common/guac_surface.h | 16 --- 2 files changed, 32 insertions(+), 256 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 53e49011..656ce4a8 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -261,182 +261,7 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co } -/** - * Actual method which flushes a bitmap described by the dirty rectangle - * on the socket associated with the surface. - * - * The bitmap will be sent as a "jpeg" or "png" instruction based on the lossy - * flag. Certain conditions may override the lossy flag and send a lossless - * update. - * - * @param surface - * The surface whose dirty area will be flushed. - * - * @param dirty_rect - * The dirty rectangle. - * - * @param lossy - * Flag indicating whether this refresh should be lossy. - */ -static void __guac_common_surface_flush_to_bitmap_impl(guac_common_surface* surface, - guac_common_rect* dirty_rect, int lossy) { - - guac_socket* socket = surface->socket; - const guac_layer* layer = surface->layer; - int send_jpeg = 0; - - /* Set the JPEG flag indicating whether this bitmap should be sent as JPEG. - * Only send as a JPEG if the dirty is larger than the minimum JPEG bitmap - * size to avoid the JPEG image compression tax. */ - if (lossy && - (dirty_rect->width * dirty_rect->height) > GUAC_SURFACE_JPEG_MIN_BITMAP_SIZE) { - - /* Check the smoothness of the dirty rectangle. If smooth, do not send - * a JPEG as it has a higher overhead than standard PNG. */ - if (!guac_common_surface_rect_is_smooth(surface, dirty_rect)) { - - send_jpeg = 1; - - /* Tweak the rectangle if it is to be sent as JPEG so the size - * matches the JPEG block size. */ - guac_common_rect max; - guac_common_rect_init(&max, 0, 0, surface->width, surface->height); - - guac_common_rect_expand_to_grid(GUAC_SURFACE_JPEG_BLOCK_SIZE, - dirty_rect, &max); - } - - } - - /* Get Cairo surface for specified rect. - * The buffer is created with 4 bytes per pixel because Cairo's 24 bit RGB - * really is 32 bit BGRx */ - unsigned char* buffer = surface->buffer + dirty_rect->y * surface->stride + dirty_rect->x * 4; - cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24, - dirty_rect->width, - dirty_rect->height, - surface->stride); - - /* Send bitmap update for the dirty rectangle */ - if (send_jpeg) { - guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer, - dirty_rect->x, dirty_rect->y, rect, - GUAC_SURFACE_JPEG_IMAGE_QUALITY); - } - else { - guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, layer, - dirty_rect->x, dirty_rect->y, rect); - } - - cairo_surface_destroy(rect); - -} - -/** - * Flushes the rectangle to the given surface's bitmap queue. There MUST be - * space within the queue. - * - * @param surface The surface queue to flush to. - * @param rect The rectangle to flush. - */ -static void __guac_common_surface_flush_rect_to_queue(guac_common_surface* surface, - const guac_common_rect* rect) { - guac_common_surface_bitmap_rect* bitmap_rect; - - /* Add new rect to queue */ - bitmap_rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]); - bitmap_rect->rect = *rect; - bitmap_rect->flushed = 0; -} - -/** - * Flushes the bitmap update currently described by a lossy rectangle within the - * given surface. - * - * Scans through the regular bitmap update queue and excludes any rectangles - * covered by the lossy rectangle. - * - * @param surface - * The surface whose lossy area will be flushed. - */ -static void __guac_common_surface_flush_lossy_bitmap( - guac_common_surface* surface) { - - if (surface->lossy_dirty) { - - guac_common_surface_bitmap_rect* current = surface->bitmap_queue; - int original_queue_length = surface->bitmap_queue_length; - - /* Identify all bitmaps in queue which are - * covered by the lossy rectangle. */ - for (int i=0; i < original_queue_length; i++) { - - int intersects = guac_common_rect_intersects(¤t->rect, - &surface->lossy_dirty_rect); - /* Complete intersection. */ - if (intersects == 2) { - - /* Exclude this from the normal refresh as it is completely - * covered by the lossy dirty rectangle. */ - current->flushed = 1; - - } - - /* Partial intersection. - * The rectangle will be split if there is room on the queue. */ - else if (intersects == 1 && - surface->bitmap_queue_length < GUAC_COMMON_SURFACE_QUEUE_SIZE-5) { - - /* Clip and split rectangle into rectangles that are outside the - * lossy rectangle which are added to the normal refresh queue. - * The remaining rectangle which overlaps with the lossy - * rectangle is marked flushed to not be refreshed in the normal - * refresh cycle. - */ - guac_common_rect split_rect; - while (guac_common_rect_clip_and_split(¤t->rect, - &surface->lossy_dirty_rect, &split_rect)) { - - /* Add new rectangle to update queue */ - __guac_common_surface_flush_rect_to_queue(surface, - &split_rect); - - } - - /* Exclude the remaining part of the dirty rectangle - * which is completely covered by the lossy dirty rectangle. */ - current->flushed = 1; - - } - current++; - - } - - /* Flush the lossy bitmap */ - __guac_common_surface_flush_to_bitmap_impl(surface, - &surface->lossy_dirty_rect, 1); - - /* Flag this area as lossy so it can be moved back to the - * dirty rect and refreshed normally when refreshed less frequently. */ - int x = surface->lossy_dirty_rect.x; - int y = surface->lossy_dirty_rect.y; - int w = (x + surface->lossy_dirty_rect.width) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int h = (y + surface->lossy_dirty_rect.height) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - x /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - y /= GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - - for (int j = y; j <= h; j++) { - for (int i = x; i <= w; i++) { - surface->lossy_rect[j][i] = 1; - } - } - - /* Clear the lossy dirty flag. */ - surface->lossy_dirty = 0; - } - -} - +#if 0 /** * Calculate the current average refresh frequency for a given area on the * surface. @@ -510,6 +335,7 @@ static unsigned int __guac_common_surface_calculate_framerate( return 0; } +#endif /** * Touch the heat map with this update rectangle, so that the update @@ -558,35 +384,6 @@ static void __guac_common_surface_touch_rect(guac_common_surface* surface, } -/** - * Expands the lossy dirty rectangle of the given surface to contain the - * rectangle described by the given coordinates. - * - * @param surface - * The surface to mark as dirty. - * - * @param rect - * The rectangle of the update which is dirtying the surface. - */ -static void __guac_common_mark_lossy_dirty(guac_common_surface* surface, - const guac_common_rect* rect) { - - /* Ignore empty rects */ - if (rect->width <= 0 || rect->height <= 0) - return; - - /* If already dirty, update existing rect */ - if (surface->lossy_dirty) { - guac_common_rect_extend(&surface->lossy_dirty_rect, rect); - } - /* Otherwise init lossy dirty rect */ - else { - surface->lossy_dirty_rect = *rect; - surface->lossy_dirty = 1; - } - -} - /** * Flushes the bitmap update currently described by the dirty rectangle within the * given surface to that surface's bitmap queue. There MUST be space within the @@ -596,15 +393,20 @@ static void __guac_common_mark_lossy_dirty(guac_common_surface* surface, */ static void __guac_common_surface_flush_to_queue(guac_common_surface* surface) { + guac_common_surface_bitmap_rect* rect; + /* Do not flush if not dirty */ if (!surface->dirty) return; /* Add new rect to queue */ - __guac_common_surface_flush_rect_to_queue(surface, &surface->dirty_rect); + rect = &(surface->bitmap_queue[surface->bitmap_queue_length++]); + rect->rect = surface->dirty_rect; + rect->flushed = 0; /* Surface now flushed */ surface->dirty = 0; + } void guac_common_surface_flush_deferred(guac_common_surface* surface) { @@ -1132,30 +934,16 @@ void guac_common_surface_draw(guac_common_surface* surface, int x, int y, cairo_ if (rect.width <= 0 || rect.height <= 0) return; - unsigned int freq = 0; - /* Update the heat map for the update rectangle. */ guac_timestamp time = guac_timestamp_current(); __guac_common_surface_touch_rect(surface, &rect, time); - /* Calculate the average framerate for this rectangle. */ - freq = __guac_common_surface_calculate_framerate(surface, &rect); + /* Flush if not combining */ + if (!__guac_common_should_combine(surface, &rect, 0)) + guac_common_surface_flush_deferred(surface); - /* If this rectangle is hot, mark lossy dirty rectangle. */ - if (freq >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY) { - __guac_common_mark_lossy_dirty(surface, &rect); - } - /* Standard refresh path */ - else { - - /* Flush if not combining */ - if (!__guac_common_should_combine(surface, &rect, 0)) - guac_common_surface_flush_deferred(surface); - - /* Always defer draws */ - __guac_common_mark_dirty(surface, &rect); - - } + /* Always defer draws */ + __guac_common_mark_dirty(surface, &rect); } @@ -1325,25 +1113,32 @@ void guac_common_surface_reset_clip(guac_common_surface* surface) { } /** - * Flushes the bitmap update currently described by the dirty rectangle within the + * Flushes the bitmap update currently described by the dirty rectangle within + * the given surface directly via an "img" instruction as PNG data. The + * resulting instructions will be sent over the socket associated with the * given surface. * - * @param surface The surface to flush. + * @param surface + * The surface to flush. */ -static void __guac_common_surface_flush_to_bitmap(guac_common_surface* surface) { +static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { if (surface->dirty) { - guac_common_rect dirty_rect; - guac_common_rect_init(&dirty_rect, - surface->dirty_rect.x, - surface->dirty_rect.y, - surface->dirty_rect.width, - surface->dirty_rect.height); + guac_socket* socket = surface->socket; + const guac_layer* layer = surface->layer; - /* Flush bitmap */ - __guac_common_surface_flush_to_bitmap_impl(surface, &dirty_rect, 0); + /* Get Cairo surface for specified rect */ + unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4; + cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24, + surface->dirty_rect.width, + surface->dirty_rect.height, + surface->stride); + /* Send PNG for rect */ + guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, layer, + surface->dirty_rect.x, surface->dirty_rect.y, rect); + cairo_surface_destroy(rect); surface->realized = 1; /* Surface is no longer dirty */ @@ -1381,9 +1176,6 @@ void guac_common_surface_flush(guac_common_surface* surface) { /* Flush final dirty rectangle to queue. */ __guac_common_surface_flush_to_queue(surface); - /* Flush the lossy bitmap to client. */ - __guac_common_surface_flush_lossy_bitmap(surface); - guac_common_surface_bitmap_rect* current = surface->bitmap_queue; int i, j; int original_queue_length; @@ -1435,7 +1227,7 @@ void guac_common_surface_flush(guac_common_surface* surface) { /* Flush as bitmap otherwise */ else { if (surface->dirty) flushed++; - __guac_common_surface_flush_to_bitmap(surface); + __guac_common_surface_flush_to_png(surface); } } diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index 50a20e2e..ff08a998 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -198,22 +198,6 @@ typedef struct guac_common_surface { */ guac_common_surface_heat_rect heat_map[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS]; - /* - * Map of areas currently refreshed lossy. - */ - int lossy_rect[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS]; - - /** - * Non-zero if this surface's lossy area is dirty and needs to be flushed, - * 0 otherwise. - */ - int lossy_dirty; - - /** - * The lossy area's dirty rectangle. - */ - guac_common_rect lossy_dirty_rect; - } guac_common_surface; /** From baf01d5524a8420da03e835f77252f2e39579687 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 01:34:32 -0700 Subject: [PATCH 15/27] GUAC-240: Flush to JPEG if dirty rect is hot. --- src/common/guac_surface.c | 59 +++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 656ce4a8..17bf7df2 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -261,7 +261,6 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co } -#if 0 /** * Calculate the current average refresh frequency for a given area on the * surface. @@ -335,7 +334,6 @@ static unsigned int __guac_common_surface_calculate_framerate( return 0; } -#endif /** * Touch the heat map with this update rectangle, so that the update @@ -1148,6 +1146,43 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { } +/** + * Flushes the bitmap update currently described by the dirty rectangle within + * the given surface directly via an "img" instruction as JPEG data. The + * resulting instructions will be sent over the socket associated with the + * given surface. + * + * @param surface + * The surface to flush. + */ +static void __guac_common_surface_flush_to_jpeg(guac_common_surface* surface) { + + if (surface->dirty) { + + guac_socket* socket = surface->socket; + const guac_layer* layer = surface->layer; + + /* Get Cairo surface for specified rect */ + unsigned char* buffer = surface->buffer + surface->dirty_rect.y * surface->stride + surface->dirty_rect.x * 4; + cairo_surface_t* rect = cairo_image_surface_create_for_data(buffer, CAIRO_FORMAT_RGB24, + surface->dirty_rect.width, + surface->dirty_rect.height, + surface->stride); + + /* Send JPEG for rect */ + guac_client_stream_jpeg(surface->client, socket, GUAC_COMP_OVER, layer, + surface->dirty_rect.x, surface->dirty_rect.y, rect, + GUAC_SURFACE_JPEG_IMAGE_QUALITY); + cairo_surface_destroy(rect); + surface->realized = 1; + + /* Surface is no longer dirty */ + surface->dirty = 0; + + } + +} + /** * Comparator for instances of guac_common_surface_bitmap_rect, the elements * which make up a surface's bitmap buffer. @@ -1225,9 +1260,23 @@ void guac_common_surface_flush(guac_common_surface* surface) { __guac_common_surface_flush_to_queue(surface); /* Flush as bitmap otherwise */ - else { - if (surface->dirty) flushed++; - __guac_common_surface_flush_to_png(surface); + else if (surface->dirty) { + + flushed++; + + /* Calculate the average framerate for the dirty rect */ + int framerate = + __guac_common_surface_calculate_framerate(surface, + &surface->dirty_rect); + + /* If this rectangle is hot, flush as JPEG */ + if (framerate >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY) + __guac_common_surface_flush_to_jpeg(surface); + + /* Otherwise, use PNG */ + else + __guac_common_surface_flush_to_png(surface); + } } From dd2e0203515bdce87a341b5600b27ca9e356fcd2 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 01:44:31 -0700 Subject: [PATCH 16/27] GUAC-240: Move JPEG optimality test into own function. --- src/common/guac_surface.c | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 17bf7df2..3b29d6eb 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -284,7 +284,7 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co * The average refresh frequency. */ static unsigned int __guac_common_surface_calculate_framerate( - guac_common_surface* surface, guac_common_rect* rect) { + guac_common_surface* surface, const guac_common_rect* rect) { int x, y; @@ -335,6 +335,31 @@ static unsigned int __guac_common_surface_calculate_framerate( } +/** + * Returns whether the given rectangle would be optimally encoded as JPEG + * rather than PNG. + * + * @param surface + * The surface to be queried. + * + * @param rect + * The rectangle to check. + * + * @return + * Non-zero if the rectangle would be optimally encoded as JPEG, zero + * otherwise. + */ +static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface, + const guac_common_rect* rect) { + + /* Calculate the average framerate for the given rect */ + int framerate = __guac_common_surface_calculate_framerate(surface, rect); + + /* JPEG is preferred if rect is hot and smooth */ + return framerate >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY; + +} + /** * Touch the heat map with this update rectangle, so that the update * frequency can be calculated later. @@ -1264,13 +1289,9 @@ void guac_common_surface_flush(guac_common_surface* surface) { flushed++; - /* Calculate the average framerate for the dirty rect */ - int framerate = - __guac_common_surface_calculate_framerate(surface, - &surface->dirty_rect); - - /* If this rectangle is hot, flush as JPEG */ - if (framerate >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY) + /* Flush as JPEG if JPEG is preferred */ + if (__guac_common_surface_should_use_jpeg(surface, + &surface->dirty_rect)) __guac_common_surface_flush_to_jpeg(surface); /* Otherwise, use PNG */ From f7cb3d56e95cdb260a3e0648da735ec6944fa770 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 01:51:03 -0700 Subject: [PATCH 17/27] GUAC-240: Update function documentation. Remove unused macros. --- src/common/guac_surface.c | 52 +++++++++++++-------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 3b29d6eb..f19a1648 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -102,20 +102,9 @@ #define GUAC_SURFACE_JPEG_IMAGE_QUALITY 90 /** - * Time (msec) between each time the surface's heat map is recalculated. + * The framerate which, if exceeded, indicates that JPEG is preferred. */ -#define GUAC_COMMON_SURFACE_HEAT_MAP_UPDATE_FREQ 2000 - -/** - * Refresh frequency threshold for when an area should be refreshed lossy. - */ -#define GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY 3 - -/** - * Time delay threshold between two updates where a lossy area will be moved - * to the non-lossy refresh pipe. - */ -#define GUAC_COMMON_SURFACE_NON_LOSSY_REFRESH_THRESHOLD 3000 +#define GUAC_COMMON_SURFACE_JPEG_FRAMERATE 3 /** * Updates the coordinates of the given rectangle to be within the bounds of @@ -262,26 +251,17 @@ static void __guac_common_mark_dirty(guac_common_surface* surface, const guac_co } /** - * Calculate the current average refresh frequency for a given area on the - * surface. + * Calculate the current average framerate for a given area on the surface. * * @param surface - * The surface on which the refresh frequency will be calculated. + * The surface on which the framerate will be calculated. * - * @param x - * The x coordinate for the area. - * - * @param y - * The y coordinate for the area. - * - * @param w - * The area width. - * - * @param h - * The area height. + * @param rect + * The rect containing the area for which the average framerate will be + * calculated. * * @return - * The average refresh frequency. + * The average framerate of the given area, in frames per second. */ static unsigned int __guac_common_surface_calculate_framerate( guac_common_surface* surface, const guac_common_rect* rect) { @@ -355,23 +335,25 @@ static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface, /* Calculate the average framerate for the given rect */ int framerate = __guac_common_surface_calculate_framerate(surface, rect); - /* JPEG is preferred if rect is hot and smooth */ - return framerate >= GUAC_COMMON_SURFACE_LOSSY_REFRESH_FREQUENCY; + /* JPEG is preferred if framerate is high enough */ + return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE; } /** - * Touch the heat map with this update rectangle, so that the update - * frequency can be calculated later. + * Updates the heat map cells which intersect the given rectangle using the + * given timestamp. This timestamp, along with timestamps from past updates, + * is used to calculate the framerate of each heat cell. * * @param surface - * The surface containing the rectangle to be updated. + * The surface containing the heat map cells to be updated. * * @param rect - * The rectangle updated. + * The rectangle containing the heat map cells to be updated. * * @param time - * The time stamp of this update. + * The timestamp to use when updating the heat map cells which intersect + * the given rectangle. */ static void __guac_common_surface_touch_rect(guac_common_surface* surface, guac_common_rect* rect, guac_timestamp time) { From b56afd8bb8504c6d9c2edee67b0368517bb540cc Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 06:29:30 -0700 Subject: [PATCH 18/27] GUAC-240: Approximate whether images will compress well with JPEG vs. PNG. --- src/common/Makefile.am | 6 +- src/common/guac_surface.c | 68 +++++++++++- src/common/guac_surface_smoothness.c | 158 --------------------------- src/common/guac_surface_smoothness.h | 43 -------- 4 files changed, 68 insertions(+), 207 deletions(-) delete mode 100644 src/common/guac_surface_smoothness.c delete mode 100644 src/common/guac_surface_smoothness.h diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 816e1a74..370df685 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -35,8 +35,7 @@ noinst_HEADERS = \ guac_pointer_cursor.h \ guac_rect.h \ guac_string.h \ - guac_surface.h \ - guac_surface_smoothness.h + guac_surface.h libguac_common_la_SOURCES = \ guac_io.c \ @@ -48,8 +47,7 @@ libguac_common_la_SOURCES = \ guac_pointer_cursor.c \ guac_rect.c \ guac_string.c \ - guac_surface.c \ - guac_surface_smoothness.c + guac_surface.c libguac_common_la_CFLAGS = \ -Werror -Wall -pedantic \ diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index f19a1648..51c06b55 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -23,7 +23,6 @@ #include "config.h" #include "guac_rect.h" #include "guac_surface.h" -#include "guac_surface_smoothness.h" #include #include @@ -313,6 +312,70 @@ static unsigned int __guac_common_surface_calculate_framerate( return 0; +} + + /** + * Guesses whether a rectangle within a particular surface would be better + * compressed as PNG or as JPEG. Positive values indicate PNG is likely to + * be superior, while negative values indicate JPEG. + * + * @param surface + * The surface containing the image data to check. + * + * @param rect + * The rect to check within the given surface. + * + * @return + * Positive values if PNG compression is likely to perform better than + * JPEG, or negative values if JPEG is likely to perform better than PNG. + */ +static int guac_common_surface_png_optimality(guac_common_surface* surface, + const guac_common_rect* rect) { + + int x, y; + + int similarity = 0; + + /* Get image/buffer metrics */ + int width = rect->width; + int height = rect->height; + int stride = surface->stride; + + /* Get buffer from surface */ + unsigned char* buffer = surface->buffer + rect->y * stride + rect->x * 4; + + /* For each row */ + for (y = 0; y < height; y++) { + + uint32_t last_pixel = -1; + uint32_t* row = (uint32_t*) buffer; + + /* For each pixel in current row */ + for (x = 0; x < width; x++) { + + /* Get next pixel */ + uint32_t current_pixel = *(row++); + + /* Update similarity according to whether pixel is identical */ + if (last_pixel != -1) { + if (current_pixel == last_pixel) + similarity++; + else + similarity--; + } + + last_pixel = current_pixel; + + } + + /* Advance to next row */ + buffer += stride; + + } + + /* Return rough approximation of optimality for PNG compression */ + return 0xFF * similarity / width / height; + } /** @@ -336,7 +399,8 @@ static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface, int framerate = __guac_common_surface_calculate_framerate(surface, rect); /* JPEG is preferred if framerate is high enough */ - return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE; + return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE + && guac_common_surface_png_optimality(surface, rect) < 0; } diff --git a/src/common/guac_surface_smoothness.c b/src/common/guac_surface_smoothness.c deleted file mode 100644 index 9300573b..00000000 --- a/src/common/guac_surface_smoothness.c +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (C) 2015 Glyptodon LLC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* - * Smoothness detection from: - * QEMU VNC display driver: tight encoding - * - * From libvncserver/libvncserver/tight.c - * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved. - * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. - * - * Copyright (C) 2010 Corentin Chary - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "guac_surface_smoothness.h" - -#include -#include -#include - -/** - * The threshold to determine an image to be smooth. - */ -#define GUAC_SURFACE_SMOOTHNESS_THRESHOLD 0 - -/** - * Width of sub-row when detecting image smoothness. - */ -#define GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH 7 - -int guac_common_surface_rect_is_smooth(guac_common_surface* surface, - guac_common_rect* rect) -{ - - /* - * Code to guess if the image in a given rectangle is smooth - * (by applying "gradient" filter or JPEG coder). - */ - int x, y, d, dx; - unsigned int c; - unsigned int stats[256]; - int pixels = 0; - int pix, left[3]; - unsigned char* buffer = surface->buffer; - int stride = surface->stride; - int w = rect->x + rect->width; - int h = rect->y + rect->height; - - /* If rect is out of bounds, bail out */ - if (rect->x < 0 || rect->y < 0 || - w > surface->width || h > surface->height) { - return 0; - } - - /* If rect is too small to process, bail out */ - if (rect->width < GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH + 1 || - rect->height < GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH + 1) { - return 0; - } - - /* Init stats array */ - memset(stats, 0, sizeof (stats)); - - for (y = rect->y, x = rect->x; y < h && x < w;) { - - /* Scan sub-sections of the surface to determine how close the colors are - * to the previous. */ - for (d = 0; - d < h - y && d < w - x - GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH; - d++) { - - for (c = 0; c < 3; c++) { - unsigned int index = (y+d)*stride + (x+d)*4 + c; - left[c] = buffer[index] & 0xFF; - } - - for (dx = 1; dx <= GUAC_SURFACE_SMOOTHNESS_DETECT_SUBROW_WIDTH; dx++) { - - for (c = 0; c < 3; c++) { - unsigned int index = (y+d)*stride + (x+d+dx)*4 + c; - pix = buffer[index] & 0xFF; - stats[abs(pix - left[c])]++; - left[c] = pix; - } - ++pixels; - } - } - - /* Advance to next section */ - if (w > h) { - x += h; - y = rect->y; - } else { - x = rect->x; - y += w; - } - } - - if (pixels == 0) { - return 1; - } - - /* 95% smooth or more */ - if (stats[0] * 33 / pixels >= 95) { - return 1; - } - - unsigned int smoothness = 0; - for (c = 1; c < 8; c++) { - smoothness += stats[c] * (c * c); - if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { - return 1; - } - } - for (; c < 256; c++) { - smoothness += stats[c] * (c * c); - } - smoothness /= (pixels * 3 - stats[0]); - - return smoothness <= GUAC_SURFACE_SMOOTHNESS_THRESHOLD; -} diff --git a/src/common/guac_surface_smoothness.h b/src/common/guac_surface_smoothness.h deleted file mode 100644 index 7bd15145..00000000 --- a/src/common/guac_surface_smoothness.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2015 Glyptodon LLC - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef __GUAC_COMMON_SURFACE_SMOOTHNESS_H -#define __GUAC_COMMON_SURFACE_SMOOTHNESS_H - -#include "guac_surface.h" - -/** - * Returns the smoothness of an area on a surface. - * - * @param surface - * The surface on which the rectangle exists. - * - * @param rect - * The rectangle to check for smoothness. - * - * @return - * 1 if rectangle is smooth, zero if not. - */ -int guac_common_surface_rect_is_smooth(guac_common_surface* surface, - guac_common_rect* rect); - -#endif From adcb887efba9446abe35c2bf72df9f44130da43a Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 08:30:23 -0700 Subject: [PATCH 19/27] GUAC-240: Correct PNG optimality calculations. --- src/common/guac_surface.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 51c06b55..7b40a8d7 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -347,17 +347,17 @@ static int guac_common_surface_png_optimality(guac_common_surface* surface, /* For each row */ for (y = 0; y < height; y++) { - uint32_t last_pixel = -1; + uint32_t last_pixel; uint32_t* row = (uint32_t*) buffer; /* For each pixel in current row */ for (x = 0; x < width; x++) { /* Get next pixel */ - uint32_t current_pixel = *(row++); + uint32_t current_pixel = *(row++) | 0xFF000000; /* Update similarity according to whether pixel is identical */ - if (last_pixel != -1) { + if (x != 0) { if (current_pixel == last_pixel) similarity++; else From b6a2de8a97a6aa1963a1f72990d88b71edc3c2d3 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 08:37:43 -0700 Subject: [PATCH 20/27] GUAC-240: Remove whitespace changes. --- src/common/Makefile.am | 40 +++++++++++++++++++-------------------- src/common/guac_surface.c | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 370df685..4f4b02c8 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -25,28 +25,28 @@ ACLOCAL_AMFLAGS = -I m4 noinst_LTLIBRARIES = libguac_common.la -noinst_HEADERS = \ - guac_io.h \ - guac_clipboard.h \ - guac_dot_cursor.h \ - guac_iconv.h \ - guac_json.h \ - guac_list.h \ - guac_pointer_cursor.h \ - guac_rect.h \ - guac_string.h \ +noinst_HEADERS = \ + guac_io.h \ + guac_clipboard.h \ + guac_dot_cursor.h \ + guac_iconv.h \ + guac_json.h \ + guac_list.h \ + guac_pointer_cursor.h \ + guac_rect.h \ + guac_string.h \ guac_surface.h -libguac_common_la_SOURCES = \ - guac_io.c \ - guac_clipboard.c \ - guac_dot_cursor.c \ - guac_iconv.c \ - guac_json.c \ - guac_list.c \ - guac_pointer_cursor.c \ - guac_rect.c \ - guac_string.c \ +libguac_common_la_SOURCES = \ + guac_io.c \ + guac_clipboard.c \ + guac_dot_cursor.c \ + guac_iconv.c \ + guac_json.c \ + guac_list.c \ + guac_pointer_cursor.c \ + guac_rect.c \ + guac_string.c \ guac_surface.c libguac_common_la_CFLAGS = \ diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 7b40a8d7..69c7fa18 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -218,7 +218,7 @@ static int __guac_common_should_combine(guac_common_surface* surface, const guac } } - + /* Otherwise, do not combine */ return 0; From b0db2c210f6566a0f004311d2aefbf9956ae7fe3 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 08:38:41 -0700 Subject: [PATCH 21/27] GUAC-240: Remove now-unused structure member. --- src/common/guac_surface.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index ff08a998..5345375d 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -187,11 +187,6 @@ typedef struct guac_common_surface { */ guac_common_surface_bitmap_rect bitmap_queue[GUAC_COMMON_SURFACE_QUEUE_SIZE]; - /** - * Last time the heat map was refreshed. - */ - guac_timestamp last_heat_map_update; - /** * A heat map keeping track of the refresh frequency of * the areas of the screen. From 2d66ae87f91c4fd25445a597be9b3863482fd94d Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 11:02:18 -0700 Subject: [PATCH 22/27] GUAC-240: Improve PNG optimality approximation algorithm (count average run length). --- src/common/guac_surface.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 69c7fa18..d5b43ead 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -334,7 +334,8 @@ static int guac_common_surface_png_optimality(guac_common_surface* surface, int x, y; - int similarity = 0; + int num_same = 0; + int num_different = 1; /* Get image/buffer metrics */ int width = rect->width; @@ -356,12 +357,12 @@ static int guac_common_surface_png_optimality(guac_common_surface* surface, /* Get next pixel */ uint32_t current_pixel = *(row++) | 0xFF000000; - /* Update similarity according to whether pixel is identical */ + /* Update same/different counts according to pixel value */ if (x != 0) { if (current_pixel == last_pixel) - similarity++; + num_same++; else - similarity--; + num_different++; } last_pixel = current_pixel; @@ -374,7 +375,7 @@ static int guac_common_surface_png_optimality(guac_common_surface* surface, } /* Return rough approximation of optimality for PNG compression */ - return 0xFF * similarity / width / height; + return 0x100 * num_same / num_different - 0x400; } From cc21092ac2742080940d4be43290794a01befa0f Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 11:11:30 -0700 Subject: [PATCH 23/27] GUAC-240: Remove unnecessary change to PNG function. --- src/common/guac_surface.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index d5b43ead..3f6d2ff1 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -1206,8 +1206,8 @@ static void __guac_common_surface_flush_to_png(guac_common_surface* surface) { surface->stride); /* Send PNG for rect */ - guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, layer, - surface->dirty_rect.x, surface->dirty_rect.y, rect); + guac_client_stream_png(surface->client, socket, GUAC_COMP_OVER, + layer, surface->dirty_rect.x, surface->dirty_rect.y, rect); cairo_surface_destroy(rect); surface->realized = 1; From 62572073b07de7a295248bed28bda66793af3761 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 16:02:19 -0700 Subject: [PATCH 24/27] GUAC-240: Dynamically allocate heat map. Throw away heat map during resize. --- src/common/guac_surface.c | 45 +++++++++++++++++++++++++++++++++------ src/common/guac_surface.h | 22 +------------------ 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 3f6d2ff1..2e6971c8 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -278,13 +278,19 @@ static unsigned int __guac_common_surface_calculate_framerate( unsigned int sum_framerate = 0; unsigned int count = 0; + /* Get start of buffer at given coordinates */ + const guac_common_surface_heat_rect* heat_row = + surface->heat_map + min_y * surface->width + min_x; + /* Iterate over all the heat map cells for the area * and calculate the average framerate */ for (y = min_y; y < max_y; y++) { - for (x = min_x; x < max_x; x++) { - const guac_common_surface_heat_rect* heat_rect = - &surface->heat_map[y][x]; + /* Get current row of heat map */ + const guac_common_surface_heat_rect* heat_rect = heat_row; + + /* For each cell in subset of row */ + for (x = min_x; x < max_x; x++) { /* Calculate indicies for latest and oldest history entries */ int oldest_entry = heat_rect->oldest_entry; @@ -301,9 +307,15 @@ static unsigned int __guac_common_surface_calculate_framerate( sum_framerate += GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE * 1000 / elapsed_time; + /* Next heat map cell */ + heat_rect++; count++; } + + /* Next heat map row */ + heat_row += surface->width; + } /* Calculate the average framerate over entire rect */ @@ -433,12 +445,18 @@ static void __guac_common_surface_touch_rect(guac_common_surface* surface, int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + /* Get start of buffer at given coordinates */ + guac_common_surface_heat_rect* heat_row = + surface->heat_map + min_y * surface->width + min_x; + /* Update all heat map cells which intersect with rectangle */ for (y = min_y; y <= max_y; y++) { - for (x = min_x; x <= max_x; x++) { - /* Get heat map cell at current location */ - guac_common_surface_heat_rect* heat_rect = &surface->heat_map[y][x]; + /* Get current row of heat map */ + guac_common_surface_heat_rect* heat_rect = heat_row; + + /* For each cell in subset of row */ + for (x = min_x; x <= max_x; x++) { /* Replace oldest entry with new timestamp */ heat_rect->history[heat_rect->oldest_entry] = time; @@ -449,7 +467,14 @@ static void __guac_common_surface_touch_rect(guac_common_surface* surface, GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE) heat_rect->oldest_entry = 0; + /* Advance to next heat map cell */ + heat_rect++; + } + + /* Next heat map row */ + heat_row += surface->width; + } } @@ -909,6 +934,9 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, surface->stride = cairo_format_stride_for_width(CAIRO_FORMAT_RGB24, w); surface->buffer = calloc(h, surface->stride); + /* Create corresponding heat map */ + surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_rect)); + /* Reset clipping rect */ guac_common_surface_reset_clip(surface); @@ -931,6 +959,7 @@ void guac_common_surface_free(guac_common_surface* surface) { if (surface->realized) guac_protocol_send_dispose(surface->socket, surface->layer); + free(surface->heat_map); free(surface->buffer); free(surface); @@ -967,6 +996,10 @@ void guac_common_surface_resize(guac_common_surface* surface, int w, int h) { /* Free old data */ free(old_buffer); + /* Allocate completely new heat map (can safely discard old stats) */ + free(surface->heat_map); + surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_rect)); + /* Resize dirty rect to fit new surface dimensions */ if (surface->dirty) { __guac_common_bound_rect(surface, &surface->dirty_rect, NULL, NULL); diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index 5345375d..86948bea 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -37,31 +37,11 @@ */ #define GUAC_COMMON_SURFACE_QUEUE_SIZE 256 -/** - * The maximum surface width; 2x WQXGA @ 16:10. - */ -#define GUAC_COMMON_SURFACE_MAX_WIDTH 5120 - -/** - * The maximum surface height; 2x WQXGA @ 16:10. - */ -#define GUAC_COMMON_SURFACE_MAX_HEIGHT 3200 - /** * Heat map square size in pixels. */ #define GUAC_COMMON_SURFACE_HEAT_MAP_CELL 64 -/** - * Heat map number of columns. - */ -#define GUAC_COMMON_SURFACE_HEAT_MAP_COLS (GUAC_COMMON_SURFACE_MAX_WIDTH / GUAC_COMMON_SURFACE_HEAT_MAP_CELL) - -/** - * Heat map number of rows. - */ -#define GUAC_COMMON_SURFACE_HEAT_MAP_ROWS (GUAC_COMMON_SURFACE_MAX_HEIGHT / GUAC_COMMON_SURFACE_HEAT_MAP_CELL) - /** * The number of entries to collect within each heat map cell. Collected * history entries are used to determine the framerate of the region associated @@ -191,7 +171,7 @@ typedef struct guac_common_surface { * A heat map keeping track of the refresh frequency of * the areas of the screen. */ - guac_common_surface_heat_rect heat_map[GUAC_COMMON_SURFACE_HEAT_MAP_ROWS][GUAC_COMMON_SURFACE_HEAT_MAP_COLS]; + guac_common_surface_heat_rect* heat_map; } guac_common_surface; From c604777622cb98245ca79433521a2f3238fad06c Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 16:04:07 -0700 Subject: [PATCH 25/27] GUAC-240: Remove unused macros. --- src/common/guac_surface.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index 2e6971c8..c4b9e75e 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -79,20 +79,6 @@ #define cairo_format_stride_for_width(format, width) (width*4) #endif -/** - * The JPEG compression minimum block size. This defines the optimal rectangle - * block size factor for JPEG compression to reduce artifacts. Usually this is - * 8 (8x8), but use 16 to reduce the occurence of ringing artifacts further. - */ -#define GUAC_SURFACE_JPEG_BLOCK_SIZE 16 - -/** - * Minimum JPEG bitmap size (area). If the bitmap is smaller than this - * threshold, it should be compressed as a PNG image to avoid the JPEG - * compression tax. - */ -#define GUAC_SURFACE_JPEG_MIN_BITMAP_SIZE 4096 - /** * The JPEG image quality ('quantization') setting to use. Range 0-100 where * 100 is the highest quality/largest file size, and 0 is the lowest From 16fd8f6c7db4564b581fd5e752b5c4e1e6838798 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 16:09:40 -0700 Subject: [PATCH 26/27] GUAC-240: Fix buffer error in calculation of framerate. Clarify naming. --- src/common/guac_surface.c | 52 +++++++++++++++++++-------------------- src/common/guac_surface.h | 19 +++++++------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index c4b9e75e..f63fa47d 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -254,18 +254,18 @@ static unsigned int __guac_common_surface_calculate_framerate( int x, y; /* Calculate minimum X/Y coordinates intersecting given rect */ - int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; /* Calculate maximum X/Y coordinates intersecting given rect */ - int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; unsigned int sum_framerate = 0; unsigned int count = 0; /* Get start of buffer at given coordinates */ - const guac_common_surface_heat_rect* heat_row = + const guac_common_surface_heat_cell* heat_row = surface->heat_map + min_y * surface->width + min_x; /* Iterate over all the heat map cells for the area @@ -273,28 +273,28 @@ static unsigned int __guac_common_surface_calculate_framerate( for (y = min_y; y < max_y; y++) { /* Get current row of heat map */ - const guac_common_surface_heat_rect* heat_rect = heat_row; + const guac_common_surface_heat_cell* heat_cell = heat_row; /* For each cell in subset of row */ for (x = min_x; x < max_x; x++) { /* Calculate indicies for latest and oldest history entries */ - int oldest_entry = heat_rect->oldest_entry; + int oldest_entry = heat_cell->oldest_entry; int latest_entry = oldest_entry - 1; if (latest_entry < 0) - latest_entry = GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE; + latest_entry = GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE - 1; /* Calculate elapsed time covering entire history for this cell */ - int elapsed_time = heat_rect->history[latest_entry] - - heat_rect->history[oldest_entry]; + int elapsed_time = heat_cell->history[latest_entry] + - heat_cell->history[oldest_entry]; /* Calculate and add framerate */ if (elapsed_time) - sum_framerate += GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE + sum_framerate += GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE * 1000 / elapsed_time; /* Next heat map cell */ - heat_rect++; + heat_cell++; count++; } @@ -424,37 +424,37 @@ static void __guac_common_surface_touch_rect(guac_common_surface* surface, int x, y; /* Calculate minimum X/Y coordinates intersecting given rect */ - int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int min_x = rect->x / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int min_y = rect->y / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; /* Calculate maximum X/Y coordinates intersecting given rect */ - int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; - int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_MAP_CELL; + int max_x = min_x + (rect->width - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; + int max_y = min_y + (rect->height - 1) / GUAC_COMMON_SURFACE_HEAT_CELL_SIZE; /* Get start of buffer at given coordinates */ - guac_common_surface_heat_rect* heat_row = + guac_common_surface_heat_cell* heat_row = surface->heat_map + min_y * surface->width + min_x; /* Update all heat map cells which intersect with rectangle */ for (y = min_y; y <= max_y; y++) { /* Get current row of heat map */ - guac_common_surface_heat_rect* heat_rect = heat_row; + guac_common_surface_heat_cell* heat_cell = heat_row; /* For each cell in subset of row */ for (x = min_x; x <= max_x; x++) { /* Replace oldest entry with new timestamp */ - heat_rect->history[heat_rect->oldest_entry] = time; + heat_cell->history[heat_cell->oldest_entry] = time; /* Update to next oldest entry */ - heat_rect->oldest_entry++; - if (heat_rect->oldest_entry >= - GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE) - heat_rect->oldest_entry = 0; + heat_cell->oldest_entry++; + if (heat_cell->oldest_entry >= + GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE) + heat_cell->oldest_entry = 0; /* Advance to next heat map cell */ - heat_rect++; + heat_cell++; } @@ -921,7 +921,7 @@ guac_common_surface* guac_common_surface_alloc(guac_client* client, surface->buffer = calloc(h, surface->stride); /* Create corresponding heat map */ - surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_rect)); + surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell)); /* Reset clipping rect */ guac_common_surface_reset_clip(surface); @@ -984,7 +984,7 @@ void guac_common_surface_resize(guac_common_surface* surface, int w, int h) { /* Allocate completely new heat map (can safely discard old stats) */ free(surface->heat_map); - surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_rect)); + surface->heat_map = calloc(w*h, sizeof(guac_common_surface_heat_cell)); /* Resize dirty rect to fit new surface dimensions */ if (surface->dirty) { diff --git a/src/common/guac_surface.h b/src/common/guac_surface.h index 86948bea..ad99af0e 100644 --- a/src/common/guac_surface.h +++ b/src/common/guac_surface.h @@ -38,22 +38,23 @@ #define GUAC_COMMON_SURFACE_QUEUE_SIZE 256 /** - * Heat map square size in pixels. + * Heat map cell size in pixels. Each side of each heat map cell will consist + * of this many pixels. */ -#define GUAC_COMMON_SURFACE_HEAT_MAP_CELL 64 +#define GUAC_COMMON_SURFACE_HEAT_CELL_SIZE 64 /** * The number of entries to collect within each heat map cell. Collected * history entries are used to determine the framerate of the region associated * with that cell. */ -#define GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE 5 +#define GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE 5 /** - * Representation of a rectangle or cell in the refresh heat map. This rectangle - * is used to keep track of how often an area on a surface is refreshed. + * Representation of a cell in the refresh heat map. This cell is used to keep + * track of how often an area on a surface is refreshed. */ -typedef struct guac_common_surface_heat_rect { +typedef struct guac_common_surface_heat_cell { /** * Timestamps of each of the last N updates covering the location @@ -63,14 +64,14 @@ typedef struct guac_common_surface_heat_rect { * pointed to by oldest_entry and proceeding through all other entries, * wrapping around if the end of the array is reached. */ - guac_timestamp history[GUAC_COMMON_SURFACE_HEAT_MAP_HISTORY_SIZE]; + guac_timestamp history[GUAC_COMMON_SURFACE_HEAT_CELL_HISTORY_SIZE]; /** * Index of the oldest entry within the history. */ int oldest_entry; -} guac_common_surface_heat_rect; +} guac_common_surface_heat_cell; /** * Representation of a bitmap update, having a rectangle of image data (stored @@ -171,7 +172,7 @@ typedef struct guac_common_surface { * A heat map keeping track of the refresh frequency of * the areas of the screen. */ - guac_common_surface_heat_rect* heat_map; + guac_common_surface_heat_cell* heat_map; } guac_common_surface; From 5dc5a9dbca5cfb8c5fdd364fcdd6e516066e48e1 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Mon, 17 Aug 2015 16:14:57 -0700 Subject: [PATCH 27/27] GUAC-240: Rename optimality function to match convention used elsewhere in this file. --- src/common/guac_surface.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/guac_surface.c b/src/common/guac_surface.c index f63fa47d..fce6902c 100644 --- a/src/common/guac_surface.c +++ b/src/common/guac_surface.c @@ -327,7 +327,7 @@ static unsigned int __guac_common_surface_calculate_framerate( * Positive values if PNG compression is likely to perform better than * JPEG, or negative values if JPEG is likely to perform better than PNG. */ -static int guac_common_surface_png_optimality(guac_common_surface* surface, +static int __guac_common_surface_png_optimality(guac_common_surface* surface, const guac_common_rect* rect) { int x, y; @@ -399,7 +399,7 @@ static int __guac_common_surface_should_use_jpeg(guac_common_surface* surface, /* JPEG is preferred if framerate is high enough */ return framerate >= GUAC_COMMON_SURFACE_JPEG_FRAMERATE - && guac_common_surface_png_optimality(surface, rect) < 0; + && __guac_common_surface_png_optimality(surface, rect) < 0; }