quartz: honor more source image properties in _gdk_quartz_image_copy_to_image()

Particularly look at the source image's byte order when making a
screenshot of the "root window" (which is the window stack). Make
the code more generic so it can handle all sorts of pixel formats.

gdk/quartz/gdkimage-quartz.c

@@ -134,6 +134,14 @@ _gdk_quartz_image_copy_to_image (GdkDrawable *drawable,
       guchar *data;
       int x, y;
       NSSize size;
+      NSBitmapFormat format;
+      gboolean has_alpha;
+      gint bpp;
+      gint r_byte = 0;
+      gint g_byte = 1;
+      gint b_byte = 2;
+      gint a_byte = 3;
+      gboolean le_image_data = FALSE;
 
       if (GDK_WINDOW_IMPL_QUARTZ (drawable) == GDK_WINDOW_IMPL_QUARTZ (GDK_WINDOW_OBJECT (_gdk_root)->impl))
         {
@@ -143,6 +151,23 @@ _gdk_quartz_image_copy_to_image (GdkDrawable *drawable,
                                                                kCGWindowListOptionOnScreenOnly,
                                                                kCGNullWindowID,
                                                                kCGWindowImageDefault);
+
+          /* HACK: the NSBitmapImageRep does not copy and convert
+           * CGImageRef's data so it matches what NSBitmapImageRep can
+           * express in its API (which is RGBA and ARGB, premultiplied
+           * and unpremultiplied), it only references the CGImageRef.
+           * Therefore we need to do the host byte swapping ourselves.
+           */
+          if (CGImageGetBitmapInfo (root_image_ref) & kCGBitmapByteOrder32Little)
+            {
+              r_byte = 3;
+              g_byte = 2;
+              b_byte = 1;
+              a_byte = 0;
+
+              le_image_data = TRUE;
+            }
+
           rep = [[NSBitmapImageRep alloc] initWithCGImage: root_image_ref];
           CGImageRelease (root_image_ref);
         }
@@ -161,6 +186,21 @@ _gdk_quartz_image_copy_to_image (GdkDrawable *drawable,
 
       data = [rep bitmapData];
       size = [rep size];
+      format = [rep bitmapFormat];
+      has_alpha = [rep hasAlpha];
+      bpp = [rep bitsPerPixel] / 8;
+
+      /* MORE HACK: AlphaFirst seems set for le_image_data, which is
+       * technically correct, but really apple, are you kidding, it's
+       * in fact ABGR, not ARGB as promised in NSBitmapImageRep's API.
+       */
+      if (!le_image_data && (format & NSAlphaFirstBitmapFormat))
+        {
+          r_byte = 1;
+          g_byte = 2;
+          b_byte = 3;
+          a_byte = 0;
+        }
 
       for (y = 0; y < size.height; y++)
         {
@@ -168,27 +208,38 @@ _gdk_quartz_image_copy_to_image (GdkDrawable *drawable,
 
           for (x = 0; x < size.width; x++)
             {
+              guchar r = src[r_byte];
+              guchar g = src[g_byte];
+              guchar b = src[b_byte];
               gint32 pixel;
 
-              if ([rep hasAlpha])
+              if (has_alpha)
                 {
-                  if (image->byte_order == GDK_LSB_FIRST)
+                  guchar alpha = src[a_byte];
-                    pixel = src[3] | src[2] << 8 | src[1] << 16 | src[0] << 24;
-                  else
-                    pixel = src[3] << 24 | src[2] << 16 | src[1] << 8 | src[0];
 
-                  src += [rep bitsPerPixel] / 8;
+                  /* unpremultiply if alpha > 0 */
+                  if (! (format & NSAlphaNonpremultipliedBitmapFormat) && alpha)
+                    {
+                      r = r * 255 / alpha;
+                      g = g * 255 / alpha;
+                      b = b * 255 / alpha;
+                    }
+
+                  if (image->byte_order == GDK_MSB_FIRST)
+                    pixel = alpha | b << 8 | g << 16 | r << 24;
+                  else
+                    pixel = alpha << 24 | b << 16 | g << 8 | r;
                 }
               else
                 {
-                  if (image->byte_order == GDK_LSB_FIRST)
+                  if (image->byte_order == GDK_MSB_FIRST)
-                    pixel = src[3] | src[2] << 8 |src[1] << 16;
+                    pixel = b | g << 8 | r << 16;
                   else
-                    pixel = src[3] << 16 | src[2] << 8 |src[1];
+                    pixel = b << 16 | g << 8 | r;
-
-                  src += [rep bitsPerPixel] / 8;
                 }
 
+              src += bpp;
+
               gdk_image_put_pixel (image, dest_x + x, dest_y + y, pixel);
             }
         }