On 6 Oct 2022, at 8:15 pm, Akihiko Odaki <akihiko.od...@gmail.com> wrote:
Thanks Peter and Elliot,
Unfortunately Patchew seems to have failed to apply the patch to the
current master. It would be nice if you rebase it to the current
master.
Actually I have a patch to add hardware support to ui/cocoa, but I
have not submitted to the mailing list because it depends on a number
of other patches:
https://github.com/akihikodaki/qemu/commit/34199fcd4080ce8c705b46df26fdf02966b1610c
My patch avoided using CGWarpMouseCursorPosition because of its
quirks. I'd like to test your patch by myself to see if it avoids them
properly for my own workloads.
I have also added some comments to the patch. Please see the below.
Regards,
Akihiko Odaki
On Wed, Oct 5, 2022 at 12:39 AM Peter Maydell <peter.mayd...@linaro.org> wrote:
Ccing Akihiko to see if he wants to review this cocoa ui frontend
patch.
also available at:
https://lore.kernel.org/qemu-devel/54930451-d85f-4ce0-9a45-b3478c5a6...@www.fastmail.com/
I can confirm that the patch does build, but I don't have any
interesting graphics-using test images to hand to test with.
thanks
-- PMM
On Thu, 4 Aug 2022 at 07:28, Elliot Nunn <ell...@nunn.io> wrote:
Implement dpy_cursor_define() and dpy_mouse_set() on macOS.
The main benefit is from dpy_cursor_define: in absolute pointing mode, the
host can redraw the cursor on the guest's behalf much faster than the guest
can itself.
To provide the programmatic movement expected from a hardware cursor,
dpy_mouse_set is also implemented.
Tricky cases are handled:
- dpy_mouse_set() avoids rounded window corners.
- The sometimes-delay between warping the cursor and an affected mouse-move
event is accounted for.
- Cursor bitmaps are nearest-neighbor scaled to Retina size.
Signed-off-by: Elliot Nunn <ell...@nunn.io>
---
ui/cocoa.m | 263 ++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 240 insertions(+), 23 deletions(-)
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 5a8bd5dd84..f9d54448e4 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -85,12 +85,20 @@ static void cocoa_switch(DisplayChangeListener *dcl,
static void cocoa_refresh(DisplayChangeListener *dcl);
+static void cocoa_mouse_set(DisplayChangeListener *dcl,
+ int x, int y, int on);
+
+static void cocoa_cursor_define(DisplayChangeListener *dcl,
+ QEMUCursor *c);
+
static NSWindow *normalWindow;
static const DisplayChangeListenerOps dcl_ops = {
.dpy_name = "cocoa",
.dpy_gfx_update = cocoa_update,
.dpy_gfx_switch = cocoa_switch,
.dpy_refresh = cocoa_refresh,
+ .dpy_mouse_set = cocoa_mouse_set,
+ .dpy_cursor_define = cocoa_cursor_define,
};
static DisplayChangeListener dcl = {
.ops = &dcl_ops,
@@ -313,6 +321,13 @@ @interface QemuCocoaView : NSView
BOOL isFullscreen;
BOOL isAbsoluteEnabled;
CFMachPortRef eventsTap;
+ NSCursor *guestCursor;
+ BOOL cursorHiddenByMe;
Who is "Me" here?
+ BOOL guestCursorVis;
+ int guestCursorX, guestCursorY;
+ int lastWarpX, lastWarpY;
+ int warpDeltaX, warpDeltaY;
+ BOOL ignoreNextMouseMove;
}
- (void) switchSurface:(pixman_image_t *)image;
- (void) grabMouse;
@@ -323,6 +338,10 @@ - (void) handleMonitorInput:(NSEvent *)event;
- (bool) handleEvent:(NSEvent *)event;
- (bool) handleEventLocked:(NSEvent *)event;
- (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled;
+- (void) cursorDefine:(NSCursor *)cursor;
+- (void) mouseSetX:(int)x Y:(int)y on:(int)on;
+- (void) setCursorAppearance;
+- (void) setCursorPosition;
/* The state surrounding mouse grabbing is potentially confusing.
* isAbsoluteEnabled tracks qemu_input_is_absolute() [ie "is the emulated
* pointing device an absolute-position one?"], but is only updated on
@@ -432,22 +451,6 @@ - (CGPoint) screenLocationOfEvent:(NSEvent *)ev
}
}
-- (void) hideCursor
-{
- if (!cursor_hide) {
- return;
- }
- [NSCursor hide];
-}
-
-- (void) unhideCursor
-{
- if (!cursor_hide) {
- return;
- }
- [NSCursor unhide];
-}
-
- (void) drawRect:(NSRect) rect
{
COCOA_DEBUG("QemuCocoaView: drawRect\n");
@@ -635,6 +638,8 @@ - (void) switchSurface:(pixman_image_t *)image
screen.height = h;
[self setContentDimensions];
[self setFrame:NSMakeRect(cx, cy, cw, ch)];
+ [self setCursorAppearance];
Why do you need to set cursor appearance when resizing the screen?
+ [self setCursorPosition];
}
// update screenBuffer
@@ -681,6 +686,7 @@ - (void) toggleFullScreen:(id)sender
styleMask:NSWindowStyleMaskBorderless
backing:NSBackingStoreBuffered
defer:NO];
+ [fullScreenWindow disableCursorRects];
[fullScreenWindow setAcceptsMouseMovedEvents: YES];
[fullScreenWindow setHasShadow:NO];
[fullScreenWindow setBackgroundColor: [NSColor blackColor]];
@@ -812,6 +818,7 @@ - (bool) handleEventLocked:(NSEvent *)event
int buttons = 0;
int keycode = 0;
bool mouse_event = false;
+ bool mousemoved_event = false;
// Location of event in virtual screen coordinates
NSPoint p = [self screenLocationOfEvent:event];
NSUInteger modifiers = [event modifierFlags];
@@ -1023,6 +1030,7 @@ - (bool) handleEventLocked:(NSEvent *)event
}
}
mouse_event = true;
+ mousemoved_event = true;
break;
case NSEventTypeLeftMouseDown:
buttons |= MOUSE_EVENT_LBUTTON;
@@ -1039,14 +1047,17 @@ - (bool) handleEventLocked:(NSEvent *)event
case NSEventTypeLeftMouseDragged:
buttons |= MOUSE_EVENT_LBUTTON;
mouse_event = true;
+ mousemoved_event = true;
break;
case NSEventTypeRightMouseDragged:
buttons |= MOUSE_EVENT_RBUTTON;
mouse_event = true;
+ mousemoved_event = true;
break;
case NSEventTypeOtherMouseDragged:
buttons |= MOUSE_EVENT_MBUTTON;
mouse_event = true;
+ mousemoved_event = true;
break;
case NSEventTypeLeftMouseUp:
mouse_event = true;
@@ -1121,7 +1132,12 @@ - (bool) handleEventLocked:(NSEvent *)event
qemu_input_update_buttons(dcl.con, bmap, last_buttons, buttons);
last_buttons = buttons;
}
- if (isMouseGrabbed) {
+
+ if (!isMouseGrabbed) {
+ return false;
+ }
+
+ if (mousemoved_event) {
if (isAbsoluteEnabled) {
/* Note that the origin for Cocoa mouse coords is bottom left,
not top left.
* The check on screenContainsPoint is to avoid sending out of
range values for
@@ -1132,11 +1148,38 @@ - (bool) handleEventLocked:(NSEvent *)event
qemu_input_queue_abs(dcl.con, INPUT_AXIS_Y, screen.height
- p.y, 0, screen.height);
}
} else {
- qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, (int)[event
deltaX]);
- qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, (int)[event
deltaY]);
+ if (ignoreNextMouseMove) {
+ // Discard the first mouse-move event after a grab, because
+ // it includes the warp delta from an unknown initial
position.
+ ignoreNextMouseMove = NO;
+ warpDeltaX = warpDeltaY = 0;
+ } else {
+ // Correct subsequent events to remove the known warp
delta.
+ // The warp delta is sometimes late to be reported, so
never
+ // allow the delta compensation to alter the direction.
+ int dX = (int)[event deltaX];
+ int dY = (int)[event deltaY];
+
+ if (dX == 0 || (dX ^ (dX - warpDeltaX)) < 0) { // Flipped
sign?
Instead, do: (dx < 0) == (dx - warpDeltaX < 0). The original flipped
sign check is dependent on an implementation-defined behavior, and a
bit difficult to understand. A decent compiler should be able to
optimize it to the bitwise operation.
+ warpDeltaX -= dX; // Save excess correction for later
+ dX = 0;
+ } else {
+ dX -= warpDeltaX; // Apply entire correction
+ warpDeltaX = 0;
+ }
+
+ if (dY == 0 || (dY ^ (dY - warpDeltaY)) < 0) {
+ warpDeltaY -= dY;
+ dY = 0;
+ } else {
+ dY -= warpDeltaY;
+ warpDeltaY = 0;
+ }
+
+ qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, dX);
+ qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, dY);
+ }
}
- } else {
- return false;
}
qemu_input_event_sync();
}
@@ -1153,9 +1196,15 @@ - (void) grabMouse
else
[normalWindow setTitle:@"QEMU - (Press ctrl + alt + g to release
Mouse)"];
}
- [self hideCursor];
CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
isMouseGrabbed = TRUE; // while isMouseGrabbed = TRUE, QemuCocoaApp sends
all events to [cocoaView handleEvent:]
+ [self setCursorAppearance];
+ [self setCursorPosition];
+
+ // We took over and warped the mouse, so ignore the next mouse-move
+ if (!isAbsoluteEnabled) {
+ ignoreNextMouseMove = YES;
+ }
It shouldn't warp the mouse when the pointing device is absolute. An
absolute pointing device, especially vdagent, is often used to
seamlessly integrate the guest and host cursors.
}
- (void) ungrabMouse
@@ -1168,9 +1217,14 @@ - (void) ungrabMouse
else
[normalWindow setTitle:@"QEMU"];
}
- [self unhideCursor];
CGAssociateMouseAndMouseCursorPosition(TRUE);
isMouseGrabbed = FALSE;
+ [self setCursorAppearance];
+
+ if (!isAbsoluteEnabled) {
+ ignoreNextMouseMove = NO;
+ warpDeltaX = warpDeltaY = 0;
+ }
}
- (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
@@ -1179,6 +1233,116 @@ - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
}
}
+
+// Indirectly called by dpy_cursor_define() in the virtual GPU
+- (void) cursorDefine:(NSCursor *)cursor {
+ guestCursor = cursor;
The old cursor is leaked here. Note that ARC is not enabled on QEMU,
unfortunately.
+ [self setCursorAppearance];
+}
+
+// Indirectly called by dpy_mouse_set() in the virtual GPU
+- (void) mouseSetX:(int)x Y:(int)y on:(int)on {
+ if (!on != !guestCursorVis) {
+ guestCursorVis = on;
+ [self setCursorAppearance];
+ }
+
+ if (on && (x != guestCursorX || y != guestCursorY)) {
+ guestCursorX = x;
+ guestCursorY = y;
+ [self setCursorPosition];
+ }
+}
+
+// Change the cursor image to the default, the guest cursor bitmap or hidden.
+// Said to be an expensive operation on macOS Monterey, so use sparingly.
+- (void) setCursorAppearance {
+ NSCursor *cursor = NULL; // NULL means hidden
+
+ if (!isMouseGrabbed) {
+ cursor = [NSCursor arrowCursor];
+ } else if (!guestCursor && !cursor_hide) {
+ cursor = [NSCursor arrowCursor];
+ } else if (guestCursorVis && guestCursor) {
+ cursor = guestCursor;
+ } else {
+ cursor = NULL;
+ }
+
+ if (cursor != NULL) {
+ [cursor set];
+
+ if (cursorHiddenByMe) {
+ [NSCursor unhide];
+ cursorHiddenByMe = NO;
+ }
+ } else {
+ if (!cursorHiddenByMe) {
+ [NSCursor hide];
+ cursorHiddenByMe = YES;
+ }
+ }
+}
+
+// Move the cursor within the virtual screen
+- (void) setCursorPosition {
+ // Ignore the guest's request if the cursor belongs to Cocoa
+ if (!isMouseGrabbed || isAbsoluteEnabled) {
+ return;
+ }
+
+ // Get guest screen rect in Cocoa coordinates (bottom-left origin).
+ NSRect virtualScreen = [[self window] convertRectToScreen:[self frame]];
+
+ // Convert to top-left origin.
+ NSInteger hostScreenH = [NSScreen screens][0].frame.size.height;
+ int scrX = virtualScreen.origin.x;
+ int scrY = hostScreenH - virtualScreen.origin.y -
virtualScreen.size.height;
+ int scrW = virtualScreen.size.width;
+ int scrH = virtualScreen.size.height;
+
+ int cursX = scrX + guestCursorX;
+ int cursY = scrY + guestCursorY;
+
+ // Clip to edges
+ cursX = MIN(MAX(scrX, cursX), scrX + scrW - 1);
+ cursY = MIN(MAX(scrY, cursY), scrY + scrH - 1);
+
+ // Move diagonally towards the center to avoid rounded window corners.
+ // Limit the number of hit-tests and discard failed attempts.
+ int betterX = cursX, betterY = cursY;
+ for (int i=0; i<16; i++) {
+ if ([NSWindow windowNumberAtPoint:NSMakePoint(betterX, hostScreenH -
betterY)
+ belowWindowWithWindowNumber:0] == self.window.windowNumber) {
+ cursX = betterX;
+ cursY = betterY;
+ break;
+ };
+
+ if (betterX < scrX + scrW/2) {
+ betterX++;
+ } else {
+ betterX--;
+ }
+
+ if (betterY < scrY + scrH/2) {
+ betterY++;
+ } else {
+ betterY--;
+ }
+ }
+
+ // Subtract this warp delta from the next NSEventTypeMouseMoved.
+ // These are in down-is-positive coords, same as NSEvent deltaX/deltaY.
+ warpDeltaX += cursX - lastWarpX;
+ warpDeltaY += cursY - lastWarpY;
+
+ CGWarpMouseCursorPosition(NSMakePoint(cursX, cursY));
+
+ lastWarpX = cursX;
+ lastWarpY = cursY;
+}
+
- (BOOL) isMouseGrabbed {return isMouseGrabbed;}
- (BOOL) isAbsoluteEnabled {return isAbsoluteEnabled;}
- (float) cdx {return cdx;}
@@ -1251,6 +1415,7 @@ - (id) init
error_report("(cocoa) can't create window");
exit(1);
}
+ [normalWindow disableCursorRects];
[normalWindow setAcceptsMouseMovedEvents:YES];
[normalWindow setTitle:@"QEMU"];
[normalWindow setContentView:cocoaView];
@@ -2123,6 +2288,58 @@ static void cocoa_display_init(DisplayState *ds,
DisplayOptions *opts)
qemu_clipboard_peer_register(&cbpeer);
}
+static void cocoa_mouse_set(DisplayChangeListener *dcl, int x, int y, int on) {
Put { for a function in a new line. See docs/devel/style.rst.
+ dispatch_async(dispatch_get_main_queue(), ^{
+ [cocoaView mouseSetX:x Y:y on:on];
+ });
+}
+
+// Convert QEMUCursor to NSCursor, then call cursorDefine
+static void cocoa_cursor_define(DisplayChangeListener *dcl, QEMUCursor
*cursor) {
+ CFDataRef cfdata = CFDataCreate(
+ /*allocator*/ NULL,
+ /*bytes*/ (void *)cursor->data,
+ /*length*/ sizeof(uint32_t) * cursor->width * cursor->height);
+
+ CGDataProviderRef dataprovider = CGDataProviderCreateWithCFData(cfdata);
+
+ CGImageRef cgimage = CGImageCreate(
+ cursor->width, cursor->height,
+ /*bitsPerComponent*/ 8,
+ /*bitsPerPixel*/ 32,
+ /*bytesPerRow*/ sizeof(uint32_t) * cursor->width,
+ /*colorspace*/ CGColorSpaceCreateWithName(kCGColorSpaceSRGB),
+ /*bitmapInfo*/ kCGBitmapByteOrder32Host | kCGImageAlphaLast,
+ /*provider*/ dataprovider,
+ /*decode*/ NULL,
+ /*shouldInterpolate*/ FALSE,
+ /*intent*/ kCGRenderingIntentDefault);
+
+ NSImage *unscaled = [[NSImage alloc] initWithCGImage:cgimage
size:NSZeroSize];
+
+ CFRelease(cfdata);
+ CGDataProviderRelease(dataprovider);
+ CGImageRelease(cgimage);
+
+ // Nearest-neighbor scale to the possibly "Retina" cursor size
+ NSImage *scaled = [NSImage
+ imageWithSize:NSMakeSize(cursor->width, cursor->height)
+ flipped:NO
+ drawingHandler:^BOOL(NSRect dest) {
+ [NSGraphicsContext currentContext].imageInterpolation =
NSImageInterpolationNone;
+ [unscaled drawInRect:dest];
+ return YES;
+ }];
unscaled and scaled are leaked.
+
+ NSCursor *nscursor = [[NSCursor alloc]
+ initWithImage:scaled
+ hotSpot:NSMakePoint(cursor->hot_x, cursor->hot_y)];
+
+ dispatch_async(dispatch_get_main_queue(), ^{
+ [cocoaView cursorDefine:nscursor];
+ });
+}
+
static QemuDisplay qemu_display_cocoa = {
.type = DISPLAY_TYPE_COCOA,
.init = cocoa_display_init,
