From d16ba33dee321bebc462140899a45e3593a86d85 Mon Sep 17 00:00:00 2001 From: Michael Jumper Date: Sat, 6 Feb 2021 15:10:50 -0800 Subject: [PATCH] GUACAMOLE-1204: Add support for including touch events within session recordings. --- src/common/common/recording.h | 57 ++++++++++++++++++++++++++- src/common/recording.c | 15 ++++++- src/libguac/guacamole/protocol.h | 47 ++++++++++++++++++++++ src/libguac/protocol.c | 31 +++++++++++++++ src/protocols/kubernetes/kubernetes.c | 1 + src/protocols/rdp/input.c | 5 +++ src/protocols/rdp/rdp.c | 1 + src/protocols/rdp/settings.c | 13 ++++++ src/protocols/rdp/settings.h | 8 ++++ src/protocols/ssh/ssh.c | 1 + src/protocols/telnet/telnet.c | 1 + src/protocols/vnc/vnc.c | 1 + 12 files changed, 179 insertions(+), 2 deletions(-) diff --git a/src/common/common/recording.h b/src/common/common/recording.h index b0278f01..e090f1de 100644 --- a/src/common/common/recording.h +++ b/src/common/common/recording.h @@ -71,6 +71,15 @@ typedef struct guac_common_recording { */ int include_mouse; + /** + * Non-zero if multi-touch events should be included in the session + * recording, zero otherwise. Depending on whether the remote desktop will + * automatically provide graphical feedback for touches, including touch + * events may be necessary for multi-touch interactions to be rendered in + * any resulting video. + */ + int include_touch; + /** * Non-zero if keys pressed and released should be included in the session * recording, zero otherwise. Including key events within the recording may @@ -119,6 +128,13 @@ typedef struct guac_common_recording { * otherwise. Including mouse state is necessary for the mouse cursor to be * rendered in any resulting video. * + * @param include_touch + * Non-zero if touch events should be included in the session recording, + * zero otherwise. Depending on whether the remote desktop will + * automatically provide graphical feedback for touches, including touch + * events may be necessary for multi-touch interactions to be rendered in + * any resulting video. + * * @param include_keys * Non-zero if keys pressed and released should be included in the session * recording, zero otherwise. Including key events within the recording may @@ -133,7 +149,8 @@ typedef struct guac_common_recording { */ guac_common_recording* guac_common_recording_create(guac_client* client, const char* path, const char* name, int create_path, - int include_output, int include_mouse, int include_keys); + int include_output, int include_mouse, int include_touch, + int include_keys); /** * Frees the resources associated with the given in-progress recording. Note @@ -174,6 +191,44 @@ void guac_common_recording_free(guac_common_recording* recording); void guac_common_recording_report_mouse(guac_common_recording* recording, int x, int y, int button_mask); +/** + * Reports the current state of a touch contact within the recording. + * + * @param recording + * The guac_common_recording associated with the touch contact that + * has changed state. + * + * @param id + * An arbitrary integer ID which uniquely identifies this contact relative + * to other active contacts. + * + * @param x + * The X coordinate of the center of the touch contact. + * + * @param y + * The Y coordinate of the center of the touch contact. + * + * @param x_radius + * The X radius of the ellipse covering the general area of the touch + * contact, in pixels. + * + * @param y_radius + * The Y radius of the ellipse covering the general area of the touch + * contact, in pixels. + * + * @param angle + * The rough angle of clockwise rotation of the general area of the touch + * contact, in degrees. + * + * @param force + * The relative force exerted by the touch contact, where 0 is no force + * (the touch has been lifted) and 1 is maximum force (the maximum amount + * of force representable by the device). + */ +void guac_common_recording_report_touch(guac_common_recording* recording, + int id, int x, int y, int x_radius, int y_radius, + double angle, double force); + /** * Reports a change in the state of an individual key within the recording. * diff --git a/src/common/recording.c b/src/common/recording.c index b4ad2193..01a212b7 100644 --- a/src/common/recording.c +++ b/src/common/recording.c @@ -137,7 +137,8 @@ static int guac_common_recording_open(const char* path, guac_common_recording* guac_common_recording_create(guac_client* client, const char* path, const char* name, int create_path, - int include_output, int include_mouse, int include_keys) { + int include_output, int include_mouse, int include_touch, + int include_keys) { char filename[GUAC_COMMON_RECORDING_MAX_NAME_LENGTH]; @@ -165,6 +166,7 @@ guac_common_recording* guac_common_recording_create(guac_client* client, recording->socket = guac_socket_open(fd); recording->include_output = include_output; recording->include_mouse = include_mouse; + recording->include_touch = include_touch; recording->include_keys = include_keys; /* Replace client socket with wrapped recording socket only if including @@ -203,6 +205,17 @@ void guac_common_recording_report_mouse(guac_common_recording* recording, } +void guac_common_recording_report_touch(guac_common_recording* recording, + int id, int x, int y, int x_radius, int y_radius, + double angle, double force) { + + /* Report touches only if recording should contain touch events */ + if (recording->include_touch) + guac_protocol_send_touch(recording->socket, id, x, y, + x_radius, y_radius, angle, force, guac_timestamp_current()); + +} + void guac_common_recording_report_key(guac_common_recording* recording, int keysym, int pressed) { diff --git a/src/libguac/guacamole/protocol.h b/src/libguac/guacamole/protocol.h index d23da502..362351c5 100644 --- a/src/libguac/guacamole/protocol.h +++ b/src/libguac/guacamole/protocol.h @@ -209,6 +209,53 @@ int vguac_protocol_send_log(guac_socket* socket, const char* format, int guac_protocol_send_mouse(guac_socket* socket, int x, int y, int button_mask, guac_timestamp timestamp); +/** + * Sends a touch instruction over the given guac_socket connection. + * + * If an error occurs sending the instruction, a non-zero value is + * returned, and guac_error is set appropriately. + * + * @param socket + * The guac_socket connection to use. + * + * @param id + * An arbitrary integer ID which uniquely identifies this contact relative + * to other active contacts. + * + * @param x + * The X coordinate of the center of the touch contact. + * + * @param y + * The Y coordinate of the center of the touch contact. + * + * @param x_radius + * The X radius of the ellipse covering the general area of the touch + * contact, in pixels. + * + * @param y_radius + * The Y radius of the ellipse covering the general area of the touch + * contact, in pixels. + * + * @param angle + * The rough angle of clockwise rotation of the general area of the touch + * contact, in degrees. + * + * @param force + * The relative force exerted by the touch contact, where 0 is no force + * (the touch has been lifted) and 1 is maximum force (the maximum amount + * of force representable by the device). + * + * @param timestamp + * The server timestamp (in milliseconds) at the point in time this touch + * event was acknowledged. + * + * @return + * Zero on success, non-zero on error. + */ +int guac_protocol_send_touch(guac_socket* socket, int id, int x, int y, + int x_radius, int y_radius, double angle, double force, + guac_timestamp timestamp); + /** * Sends a nest instruction over the given guac_socket connection. * diff --git a/src/libguac/protocol.c b/src/libguac/protocol.c index 354adfa1..1c53c200 100644 --- a/src/libguac/protocol.c +++ b/src/libguac/protocol.c @@ -820,6 +820,37 @@ int guac_protocol_send_mouse(guac_socket* socket, int x, int y, } +int guac_protocol_send_touch(guac_socket* socket, int id, int x, int y, + int x_radius, int y_radius, double angle, double force, + guac_timestamp timestamp) { + + int ret_val; + + guac_socket_instruction_begin(socket); + ret_val = + guac_socket_write_string(socket, "5.touch,") + || __guac_socket_write_length_int(socket, id) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, x) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, y) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, x_radius) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, y_radius) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_double(socket, angle) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_double(socket, force) + || guac_socket_write_string(socket, ",") + || __guac_socket_write_length_int(socket, timestamp) + || guac_socket_write_string(socket, ";"); + + guac_socket_instruction_end(socket); + return ret_val; + +} + int guac_protocol_send_move(guac_socket* socket, const guac_layer* layer, const guac_layer* parent, int x, int y, int z) { diff --git a/src/protocols/kubernetes/kubernetes.c b/src/protocols/kubernetes/kubernetes.c index 7158a989..1998d13b 100644 --- a/src/protocols/kubernetes/kubernetes.c +++ b/src/protocols/kubernetes/kubernetes.c @@ -234,6 +234,7 @@ void* guac_kubernetes_client_thread(void* data) { settings->create_recording_path, !settings->recording_exclude_output, !settings->recording_exclude_mouse, + 0, /* Touch events not supported */ settings->recording_include_keys); } diff --git a/src/protocols/rdp/input.c b/src/protocols/rdp/input.c index 7b31cb33..0c36d614 100644 --- a/src/protocols/rdp/input.c +++ b/src/protocols/rdp/input.c @@ -136,6 +136,11 @@ int guac_rdp_user_touch_handler(guac_user* user, int id, int x, int y, if (rdp_inst == NULL) goto complete; + /* Report touch event within recording */ + if (rdp_client->recording != NULL) + guac_common_recording_report_touch(rdp_client->recording, id, x, y, + x_radius, y_radius, angle, force); + /* Forward touch event along RDPEI channel */ guac_rdp_rdpei_touch_update(rdp_client->rdpei, id, x, y, force); diff --git a/src/protocols/rdp/rdp.c b/src/protocols/rdp/rdp.c index f0e38f99..db62bc4b 100644 --- a/src/protocols/rdp/rdp.c +++ b/src/protocols/rdp/rdp.c @@ -426,6 +426,7 @@ static int guac_rdp_handle_connection(guac_client* client) { settings->create_recording_path, !settings->recording_exclude_output, !settings->recording_exclude_mouse, + !settings->recording_exclude_touch, settings->recording_include_keys); } diff --git a/src/protocols/rdp/settings.c b/src/protocols/rdp/settings.c index a5d7aebf..38064d10 100644 --- a/src/protocols/rdp/settings.c +++ b/src/protocols/rdp/settings.c @@ -104,6 +104,7 @@ const char* GUAC_RDP_CLIENT_ARGS[] = { "recording-name", "recording-exclude-output", "recording-exclude-mouse", + "recording-exclude-touch", "recording-include-keys", "create-recording-path", "resize-method", @@ -500,6 +501,13 @@ enum RDP_ARGS_IDX { */ IDX_RECORDING_EXCLUDE_MOUSE, + /** + * Whether changes to touch contact state should NOT be included in the + * session recording. Touch state is included by default, as it may be + * necessary for touch interactions to be rendered in any resulting video. + */ + IDX_RECORDING_EXCLUDE_TOUCH, + /** * Whether keys pressed and released should be included in the session * recording. Key events are NOT included by default within the recording, @@ -1042,6 +1050,11 @@ guac_rdp_settings* guac_rdp_parse_args(guac_user* user, guac_user_parse_args_boolean(user, GUAC_RDP_CLIENT_ARGS, argv, IDX_RECORDING_EXCLUDE_MOUSE, 0); + /* Parse touch exclusion flag */ + settings->recording_exclude_touch = + guac_user_parse_args_boolean(user, GUAC_RDP_CLIENT_ARGS, argv, + IDX_RECORDING_EXCLUDE_TOUCH, 0); + /* Parse key event inclusion flag */ settings->recording_include_keys = guac_user_parse_args_boolean(user, GUAC_RDP_CLIENT_ARGS, argv, diff --git a/src/protocols/rdp/settings.h b/src/protocols/rdp/settings.h index 1a6f75c1..c540fcc8 100644 --- a/src/protocols/rdp/settings.h +++ b/src/protocols/rdp/settings.h @@ -505,6 +505,14 @@ typedef struct guac_rdp_settings { */ int recording_exclude_mouse; + /** + * Non-zero if changes to touch state should NOT be included in the session + * recording, zero otherwise. Touch state is included by default, as it may + * be necessary for touch interactions to be rendered in any resulting + * video. + */ + int recording_exclude_touch; + /** * Non-zero if keys pressed and released should be included in the session * recording, zero otherwise. Key events are NOT included by default within diff --git a/src/protocols/ssh/ssh.c b/src/protocols/ssh/ssh.c index aaa5a8eb..81fb0855 100644 --- a/src/protocols/ssh/ssh.c +++ b/src/protocols/ssh/ssh.c @@ -234,6 +234,7 @@ void* ssh_client_thread(void* data) { settings->create_recording_path, !settings->recording_exclude_output, !settings->recording_exclude_mouse, + 0, /* Touch events not supported */ settings->recording_include_keys); } diff --git a/src/protocols/telnet/telnet.c b/src/protocols/telnet/telnet.c index b2b3106e..f6ea46b4 100644 --- a/src/protocols/telnet/telnet.c +++ b/src/protocols/telnet/telnet.c @@ -581,6 +581,7 @@ void* guac_telnet_client_thread(void* data) { settings->create_recording_path, !settings->recording_exclude_output, !settings->recording_exclude_mouse, + 0, /* Touch events not supported */ settings->recording_include_keys); } diff --git a/src/protocols/vnc/vnc.c b/src/protocols/vnc/vnc.c index cd09951a..aed69fb6 100644 --- a/src/protocols/vnc/vnc.c +++ b/src/protocols/vnc/vnc.c @@ -396,6 +396,7 @@ void* guac_vnc_client_thread(void* data) { settings->create_recording_path, !settings->recording_exclude_output, !settings->recording_exclude_mouse, + 0, /* Touch events not supported */ settings->recording_include_keys); }