GUACAMOLE-249: Determine FreeRDP pixel format corresponding to local Cairo surfaces based on local platform endianness.

src/protocols/rdp/color.c

@@ -28,44 +28,19 @@
 #include <freerdp/gdi/gdi.h>
 #include <winpr/wtypes.h>
 
-/**
- * Returns the integer constant used by the FreeRDP API to represent the colors
- * used by a connection having the given bit depth. These constants each have
- * corresponding PIXEL_FORMAT_* macros defined within freerdp/codec/color.h.
- *
- * @param depth
- *     The color depth which should be translated into the integer constant
- *     defined by FreeRDP's corresponding PIXEL_FORMAT_* macro.
- *
- * @return
- *     The integer value of the PIXEL_FORMAT_* macro corresponding to the
- *     given color depth.
- */
-static UINT32 guac_rdp_get_pixel_format(int depth) {
+UINT32 guac_rdp_get_native_pixel_format(BOOL alpha) {
 
-    switch (depth) {
+    uint8_t color[] = { 0x0A, 0x0B, 0x0C, 0x0D };
 
-        /* 32- and 24-bit RGB (8 bits per color component) */
-        case 32:
-        case 24:
-            return PIXEL_FORMAT_RGB24;
+    /* Local platform stores bytes in decreasing order of significance
+     * (big-endian) */
+    if (*((uint32_t*) color) == 0x0A0B0C0D)
+        return alpha ? PIXEL_FORMAT_ARGB32 : PIXEL_FORMAT_XRGB32;
 
-        /* 16-bit palette (6-bit green, 5-bit red and blue) */
-        case 16:
-            return PIXEL_FORMAT_RGB16;
-
-        /* 15-bit RGB (5 bits per color component) */
-        case 15:
-            return PIXEL_FORMAT_RGB15;
-
-        /* 8-bit palette */
-        case 8:
-            return PIXEL_FORMAT_RGB8;
-
-    }
-
-    /* Unknown format */
-    return PIXEL_FORMAT_RGB24;
+    /* Local platform stores bytes in increasing order of significance
+     * (little-endian) */
+    else
+        return alpha ? PIXEL_FORMAT_BGRA32 : PIXEL_FORMAT_BGRX32;
 
 }
 
@@ -75,8 +50,8 @@ UINT32 guac_rdp_convert_color(rdpContext* context, UINT32 color) {
     rdpGdi* gdi = context->gdi;
 
     /* Convert given color to ARGB32 */
-    return FreeRDPConvertColor(color, guac_rdp_get_pixel_format(depth),
-            PIXEL_FORMAT_ABGR32, &gdi->palette);
+    return FreeRDPConvertColor(color, gdi_get_pixel_format(depth),
+            guac_rdp_get_native_pixel_format(TRUE), &gdi->palette);
 
 }
 

src/protocols/rdp/color.h

@@ -23,6 +23,26 @@
 #include <freerdp/freerdp.h>
 #include <winpr/wtypes.h>
 
+/**
+ * Returns the FreeRDP pixel format ID corresponding to the 32-bit RGB format
+ * used by the Cairo library's image surfaces. Cairo handles colors in terms of
+ * integers in native endianness, with CAIRO_FORMAT_ARGB32 representing a color
+ * format where the alpha channel is stored in the most significant byte,
+ * followed by red, green, and blue. FreeRDP handles colors in terms of
+ * absolute byte order, with PIXEL_FORMAT_ARGB32 representing a color format
+ * where the alpha channel is in byte 0, followed by red at byte 1, etc.
+ *
+ * @param alpha
+ *     TRUE if the returned FreeRDP pixel format should correspond to Cairo's
+ *     CAIRO_FORMAT_ARGB32, FALSE if the returned format should correspond to
+ *     Cairo's CAIRO_FORMAT_RGB24.
+ *
+ * @return
+ *     The FreeRDP pixel format ID that corresponds to the 32-bit RGB format
+ *     used by the Cairo library.
+ */
+UINT32 guac_rdp_get_native_pixel_format(BOOL alpha);
+
 /**
  * Converts the given color to ARGB32. The color given may be an index
  * referring to the palette, a 16-bit or 32-bit color, etc. all depending on

src/protocols/rdp/pointer.c

@@ -19,6 +19,7 @@
 
 #include "config.h"
 #include "client.h"
+#include "color.h"
 #include "common/cursor.h"
 #include "common/display.h"
 #include "pointer.h"
@@ -47,11 +48,12 @@ BOOL guac_rdp_pointer_new(rdpContext* context, rdpPointer* pointer) {
 
     /* Convert to alpha cursor if mask data present */
     if (pointer->andMaskData && pointer->xorMaskData)
-        freerdp_image_copy_from_pointer_data(data, PIXEL_FORMAT_BGRA32,
-                0, 0, 0, pointer->width, pointer->height,
-                pointer->xorMaskData, pointer->lengthXorMask,
-                pointer->andMaskData, pointer->lengthAndMask,
-                pointer->xorBpp, &context->gdi->palette);
+        freerdp_image_copy_from_pointer_data(data,
+                guac_rdp_get_native_pixel_format(TRUE), 0, 0, 0,
+                pointer->width, pointer->height, pointer->xorMaskData,
+                pointer->lengthXorMask, pointer->andMaskData,
+                pointer->lengthAndMask, pointer->xorBpp,
+                &context->gdi->palette);
 
     /* Create surface from image data */
     surface = cairo_image_surface_create_for_data(

src/protocols/rdp/rdp.c

@@ -27,6 +27,7 @@
 #include "channels/rdpdr/rdpdr.h"
 #include "channels/rdpsnd/rdpsnd.h"
 #include "client.h"
+#include "color.h"
 #include "common/cursor.h"
 #include "common/display.h"
 #include "common/recording.h"
@@ -138,7 +139,7 @@ BOOL rdp_freerdp_pre_connect(freerdp* instance) {
     }
 
     /* Init FreeRDP internal GDI implementation */
-    if (!gdi_init(instance, PIXEL_FORMAT_BGRX32))
+    if (!gdi_init(instance, guac_rdp_get_native_pixel_format(FALSE)))
         return FALSE;
 
     /* Set up bitmap handling */