From c8216aede6c4ac41976947521d884fa010913204 Mon Sep 17 00:00:00 2001
From: Jeremy Uejio <jeremy.uejio@sun.com>
Date: Tue, 1 Jul 2008 13:37:12 -0700
Subject: [PATCH] Sun bug #6685465: Xephyr uses wrong or bad colortable in
 8-bit mode

<http://bugs.opensolaris.org/view_bug.do?bug_id=6685465>

This bug is caused by Xephyr not handling the RGB byte order correctly
of the server where Xephyr is displaying on. The previous code just
assumed that the order was RGB and did not take into account that
Xservers may use different order (such as BGR).

The fix is to add a function to calculate the byte order and bits
to shift based on the visual mask and the visual bits_per_rgb (which
is usually 8, but could be server dependent).  Since the shifts won't
change once the display connection has been made, I can cache these
values so that Xephyr doesn't have to keep recalculating them everytime
it tries to translate the Xephyr colormap entries for Xephyr clients to
the actual server colormap entries (i.e. calling the function
hostx_set_cmap_entry() repeatedly for every colormap entry).
---
 hw/kdrive/ephyr/hostx.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/hw/kdrive/ephyr/hostx.c b/hw/kdrive/ephyr/hostx.c
index 74f9f1628..171d78c9a 100644
--- a/hw/kdrive/ephyr/hostx.c
+++ b/hw/kdrive/ephyr/hostx.c
@@ -565,14 +565,40 @@ hostx_get_visual_masks (EphyrScreenInfo screen,
     }
 }
 
+static int 
+hostx_calculate_color_shift(unsigned long mask,
+			    int bits_per_rgb)
+{
+    int shift = 0;
+    while(mask) {
+	mask = mask >> bits_per_rgb;
+	if (mask) shift += bits_per_rgb;
+    }
+    return shift;
+}
+
 void
 hostx_set_cmap_entry(unsigned char idx,
 		     unsigned char r,
 		     unsigned char g,
 		     unsigned char b)
 {
-  /* XXX Will likely break for 8 on 16, not sure if this is correct */
-  HostX.cmap[idx] = (r << 16) | (g << 8) | (b);
+/* need to calculate the shifts for RGB because server could be BGR. */
+/* XXX Not sure if this is correct for 8 on 16, but this works for 8 on 24.*/
+    static int rshift, bshift, gshift = 0;
+    static int first_time = 1;
+    if (first_time) {
+	first_time = 0;
+	rshift = hostx_calculate_color_shift(HostX.visual->red_mask,
+					     HostX.visual->bits_per_rgb);
+	gshift = hostx_calculate_color_shift(HostX.visual->green_mask,
+					     HostX.visual->bits_per_rgb);
+	bshift = hostx_calculate_color_shift(HostX.visual->blue_mask,
+					     HostX.visual->bits_per_rgb);
+    }
+    HostX.cmap[idx] = ((r << rshift) & HostX.visual->red_mask) |
+		      ((g << gshift) & HostX.visual->green_mask) |
+		      ((b << bshift) & HostX.visual->blue_mask);
 }
 
 /**