[PATCH xf86-video-amdgpu] Rotate and reflect cursor hotspot position for drmModeSetCursor2

[michel@xxxxxxxxxxx (Michel Dänzer)]

From: Michel Dänzer <michel.daenzer@xxxxxxx>

We were always passing the hotspot position in the X screen coordinate
space, but drmModeSetCursor2 needs it in the CRTC coordinate space. The
wrong hotspot position would cause the kernel driver to adjust the
HW cursor position incorrectly when the hotspot position changed.

Signed-off-by: Michel Dänzer <michel.daenzer at amd.com>
---
 src/drmmode_display.c | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/drmmode_display.c b/src/drmmode_display.c
index 31aa1db..fb41975 100644
--- a/src/drmmode_display.c
+++ b/src/drmmode_display.c
@@ -1008,13 +1008,43 @@ static void drmmode_show_cursor(xf86CrtcPtr crtc)
 	if (use_set_cursor2) {
 		xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(crtc->scrn);
 		CursorPtr cursor = xf86_config->cursor;
+		int xhot = cursor->bits->xhot;
+		int yhot = cursor->bits->yhot;
 		int ret;
 
+		if (crtc->rotation != RR_Rotate_0 &&
+		    crtc->rotation != (RR_Rotate_180 | RR_Reflect_X |
+				       RR_Reflect_Y)) {
+			int t;
+
+			/* Reflect & rotate hotspot position */
+			if (crtc->rotation & RR_Reflect_X)
+				xhot = info->cursor_w - xhot - 1;
+			if (crtc->rotation & RR_Reflect_Y)
+				yhot = info->cursor_h - yhot - 1;
+
+			switch (crtc->rotation & 0xf) {
+			case RR_Rotate_90:
+				t = xhot;
+				xhot = yhot;
+				yhot = info->cursor_w - t - 1;
+				break;
+			case RR_Rotate_180:
+				xhot = info->cursor_w - xhot - 1;
+				yhot = info->cursor_h - yhot - 1;
+				break;
+			case RR_Rotate_270:
+				t = xhot;
+				xhot = info->cursor_h - yhot - 1;
+				yhot = t;
+			}
+		}
+
 		ret = drmModeSetCursor2(pAMDGPUEnt->fd,
 					drmmode_crtc->mode_crtc->crtc_id,
 					bo_handle,
 					info->cursor_w, info->cursor_h,
-					cursor->bits->xhot, cursor->bits->yhot);
+					xhot, yhot);
 		if (ret == -EINVAL)
 			use_set_cursor2 = FALSE;
 		else
-- 
2.9.3