Ccing Akihiko to see if he wants to review this cocoa ui frontend
patch.
also available at:
https://lore.kernel.org/qemu-devel/54930451-d85f-4ce0-9a45-b3478c5a6...@www.fastmail.com/
I can confirm that the patch does build, but I don't have any
interesting graphics-using test images to hand to test with.
thanks
-- PMM
On Thu, 4 Aug 2022 at 07:28, Elliot Nunn <ell...@nunn.io> wrote:
>
> Implement dpy_cursor_define() and dpy_mouse_set() on macOS.
>
> The main benefit is from dpy_cursor_define: in absolute pointing mode, the
> host can redraw the cursor on the guest's behalf much faster than the guest
> can itself.
>
> To provide the programmatic movement expected from a hardware cursor,
> dpy_mouse_set is also implemented.
>
> Tricky cases are handled:
> - dpy_mouse_set() avoids rounded window corners.
> - The sometimes-delay between warping the cursor and an affected mouse-move
> event is accounted for.
> - Cursor bitmaps are nearest-neighbor scaled to Retina size.
>
> Signed-off-by: Elliot Nunn <ell...@nunn.io>
> ---
> ui/cocoa.m | 263 ++++++++++++++++++++++++++++++++++++++++++++++++-----
> 1 file changed, 240 insertions(+), 23 deletions(-)
>
> diff --git a/ui/cocoa.m b/ui/cocoa.m
> index 5a8bd5dd84..f9d54448e4 100644
> --- a/ui/cocoa.m
> +++ b/ui/cocoa.m
> @@ -85,12 +85,20 @@ static void cocoa_switch(DisplayChangeListener *dcl,
>
> static void cocoa_refresh(DisplayChangeListener *dcl);
>
> +static void cocoa_mouse_set(DisplayChangeListener *dcl,
> + int x, int y, int on);
> +
> +static void cocoa_cursor_define(DisplayChangeListener *dcl,
> + QEMUCursor *c);
> +
> static NSWindow *normalWindow;
> static const DisplayChangeListenerOps dcl_ops = {
> .dpy_name = "cocoa",
> .dpy_gfx_update = cocoa_update,
> .dpy_gfx_switch = cocoa_switch,
> .dpy_refresh = cocoa_refresh,
> + .dpy_mouse_set = cocoa_mouse_set,
> + .dpy_cursor_define = cocoa_cursor_define,
> };
> static DisplayChangeListener dcl = {
> .ops = &dcl_ops,
> @@ -313,6 +321,13 @@ @interface QemuCocoaView : NSView
> BOOL isFullscreen;
> BOOL isAbsoluteEnabled;
> CFMachPortRef eventsTap;
> + NSCursor *guestCursor;
> + BOOL cursorHiddenByMe;
> + BOOL guestCursorVis;
> + int guestCursorX, guestCursorY;
> + int lastWarpX, lastWarpY;
> + int warpDeltaX, warpDeltaY;
> + BOOL ignoreNextMouseMove;
> }
> - (void) switchSurface:(pixman_image_t *)image;
> - (void) grabMouse;
> @@ -323,6 +338,10 @@ - (void) handleMonitorInput:(NSEvent *)event;
> - (bool) handleEvent:(NSEvent *)event;
> - (bool) handleEventLocked:(NSEvent *)event;
> - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled;
> +- (void) cursorDefine:(NSCursor *)cursor;
> +- (void) mouseSetX:(int)x Y:(int)y on:(int)on;
> +- (void) setCursorAppearance;
> +- (void) setCursorPosition;
> /* The state surrounding mouse grabbing is potentially confusing.
> * isAbsoluteEnabled tracks qemu_input_is_absolute() [ie "is the emulated
> * pointing device an absolute-position one?"], but is only updated on
> @@ -432,22 +451,6 @@ - (CGPoint) screenLocationOfEvent:(NSEvent *)ev
> }
> }
>
> -- (void) hideCursor
> -{
> - if (!cursor_hide) {
> - return;
> - }
> - [NSCursor hide];
> -}
> -
> -- (void) unhideCursor
> -{
> - if (!cursor_hide) {
> - return;
> - }
> - [NSCursor unhide];
> -}
> -
> - (void) drawRect:(NSRect) rect
> {
> COCOA_DEBUG("QemuCocoaView: drawRect\n");
> @@ -635,6 +638,8 @@ - (void) switchSurface:(pixman_image_t *)image
> screen.height = h;
> [self setContentDimensions];
> [self setFrame:NSMakeRect(cx, cy, cw, ch)];
> + [self setCursorAppearance];
> + [self setCursorPosition];
> }
>
> // update screenBuffer
> @@ -681,6 +686,7 @@ - (void) toggleFullScreen:(id)sender
> styleMask:NSWindowStyleMaskBorderless
> backing:NSBackingStoreBuffered
> defer:NO];
> + [fullScreenWindow disableCursorRects];
> [fullScreenWindow setAcceptsMouseMovedEvents: YES];
> [fullScreenWindow setHasShadow:NO];
> [fullScreenWindow setBackgroundColor: [NSColor blackColor]];
> @@ -812,6 +818,7 @@ - (bool) handleEventLocked:(NSEvent *)event
> int buttons = 0;
> int keycode = 0;
> bool mouse_event = false;
> + bool mousemoved_event = false;
> // Location of event in virtual screen coordinates
> NSPoint p = [self screenLocationOfEvent:event];
> NSUInteger modifiers = [event modifierFlags];
> @@ -1023,6 +1030,7 @@ - (bool) handleEventLocked:(NSEvent *)event
> }
> }
> mouse_event = true;
> + mousemoved_event = true;
> break;
> case NSEventTypeLeftMouseDown:
> buttons |= MOUSE_EVENT_LBUTTON;
> @@ -1039,14 +1047,17 @@ - (bool) handleEventLocked:(NSEvent *)event
> case NSEventTypeLeftMouseDragged:
> buttons |= MOUSE_EVENT_LBUTTON;
> mouse_event = true;
> + mousemoved_event = true;
> break;
> case NSEventTypeRightMouseDragged:
> buttons |= MOUSE_EVENT_RBUTTON;
> mouse_event = true;
> + mousemoved_event = true;
> break;
> case NSEventTypeOtherMouseDragged:
> buttons |= MOUSE_EVENT_MBUTTON;
> mouse_event = true;
> + mousemoved_event = true;
> break;
> case NSEventTypeLeftMouseUp:
> mouse_event = true;
> @@ -1121,7 +1132,12 @@ - (bool) handleEventLocked:(NSEvent *)event
> qemu_input_update_buttons(dcl.con, bmap, last_buttons, buttons);
> last_buttons = buttons;
> }
> - if (isMouseGrabbed) {
> +
> + if (!isMouseGrabbed) {
> + return false;
> + }
> +
> + if (mousemoved_event) {
> if (isAbsoluteEnabled) {
> /* Note that the origin for Cocoa mouse coords is bottom
> left, not top left.
> * The check on screenContainsPoint is to avoid sending out
> of range values for
> @@ -1132,11 +1148,38 @@ - (bool) handleEventLocked:(NSEvent *)event
> qemu_input_queue_abs(dcl.con, INPUT_AXIS_Y,
> screen.height - p.y, 0, screen.height);
> }
> } else {
> - qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, (int)[event
> deltaX]);
> - qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, (int)[event
> deltaY]);
> + if (ignoreNextMouseMove) {
> + // Discard the first mouse-move event after a grab,
> because
> + // it includes the warp delta from an unknown initial
> position.
> + ignoreNextMouseMove = NO;
> + warpDeltaX = warpDeltaY = 0;
> + } else {
> + // Correct subsequent events to remove the known warp
> delta.
> + // The warp delta is sometimes late to be reported, so
> never
> + // allow the delta compensation to alter the direction.
> + int dX = (int)[event deltaX];
> + int dY = (int)[event deltaY];
> +
> + if (dX == 0 || (dX ^ (dX - warpDeltaX)) < 0) { //
> Flipped sign?
> + warpDeltaX -= dX; // Save excess correction for later
> + dX = 0;
> + } else {
> + dX -= warpDeltaX; // Apply entire correction
> + warpDeltaX = 0;
> + }
> +
> + if (dY == 0 || (dY ^ (dY - warpDeltaY)) < 0) {
> + warpDeltaY -= dY;
> + dY = 0;
> + } else {
> + dY -= warpDeltaY;
> + warpDeltaY = 0;
> + }
> +
> + qemu_input_queue_rel(dcl.con, INPUT_AXIS_X, dX);
> + qemu_input_queue_rel(dcl.con, INPUT_AXIS_Y, dY);
> + }
> }
> - } else {
> - return false;
> }
> qemu_input_event_sync();
> }
> @@ -1153,9 +1196,15 @@ - (void) grabMouse
> else
> [normalWindow setTitle:@"QEMU - (Press ctrl + alt + g to release
> Mouse)"];
> }
> - [self hideCursor];
> CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
> isMouseGrabbed = TRUE; // while isMouseGrabbed = TRUE, QemuCocoaApp
> sends all events to [cocoaView handleEvent:]
> + [self setCursorAppearance];
> + [self setCursorPosition];
> +
> + // We took over and warped the mouse, so ignore the next mouse-move
> + if (!isAbsoluteEnabled) {
> + ignoreNextMouseMove = YES;
> + }
> }
>
> - (void) ungrabMouse
> @@ -1168,9 +1217,14 @@ - (void) ungrabMouse
> else
> [normalWindow setTitle:@"QEMU"];
> }
> - [self unhideCursor];
> CGAssociateMouseAndMouseCursorPosition(TRUE);
> isMouseGrabbed = FALSE;
> + [self setCursorAppearance];
> +
> + if (!isAbsoluteEnabled) {
> + ignoreNextMouseMove = NO;
> + warpDeltaX = warpDeltaY = 0;
> + }
> }
>
> - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
> @@ -1179,6 +1233,116 @@ - (void) setAbsoluteEnabled:(BOOL)tIsAbsoluteEnabled {
> CGAssociateMouseAndMouseCursorPosition(isAbsoluteEnabled);
> }
> }
> +
> +// Indirectly called by dpy_cursor_define() in the virtual GPU
> +- (void) cursorDefine:(NSCursor *)cursor {
> + guestCursor = cursor;
> + [self setCursorAppearance];
> +}
> +
> +// Indirectly called by dpy_mouse_set() in the virtual GPU
> +- (void) mouseSetX:(int)x Y:(int)y on:(int)on {
> + if (!on != !guestCursorVis) {
> + guestCursorVis = on;
> + [self setCursorAppearance];
> + }
> +
> + if (on && (x != guestCursorX || y != guestCursorY)) {
> + guestCursorX = x;
> + guestCursorY = y;
> + [self setCursorPosition];
> + }
> +}
> +
> +// Change the cursor image to the default, the guest cursor bitmap or hidden.
> +// Said to be an expensive operation on macOS Monterey, so use sparingly.
> +- (void) setCursorAppearance {
> + NSCursor *cursor = NULL; // NULL means hidden
> +
> + if (!isMouseGrabbed) {
> + cursor = [NSCursor arrowCursor];
> + } else if (!guestCursor && !cursor_hide) {
> + cursor = [NSCursor arrowCursor];
> + } else if (guestCursorVis && guestCursor) {
> + cursor = guestCursor;
> + } else {
> + cursor = NULL;
> + }
> +
> + if (cursor != NULL) {
> + [cursor set];
> +
> + if (cursorHiddenByMe) {
> + [NSCursor unhide];
> + cursorHiddenByMe = NO;
> + }
> + } else {
> + if (!cursorHiddenByMe) {
> + [NSCursor hide];
> + cursorHiddenByMe = YES;
> + }
> + }
> +}
> +
> +// Move the cursor within the virtual screen
> +- (void) setCursorPosition {
> + // Ignore the guest's request if the cursor belongs to Cocoa
> + if (!isMouseGrabbed || isAbsoluteEnabled) {
> + return;
> + }
> +
> + // Get guest screen rect in Cocoa coordinates (bottom-left origin).
> + NSRect virtualScreen = [[self window] convertRectToScreen:[self frame]];
> +
> + // Convert to top-left origin.
> + NSInteger hostScreenH = [NSScreen screens][0].frame.size.height;
> + int scrX = virtualScreen.origin.x;
> + int scrY = hostScreenH - virtualScreen.origin.y -
> virtualScreen.size.height;
> + int scrW = virtualScreen.size.width;
> + int scrH = virtualScreen.size.height;
> +
> + int cursX = scrX + guestCursorX;
> + int cursY = scrY + guestCursorY;
> +
> + // Clip to edges
> + cursX = MIN(MAX(scrX, cursX), scrX + scrW - 1);
> + cursY = MIN(MAX(scrY, cursY), scrY + scrH - 1);
> +
> + // Move diagonally towards the center to avoid rounded window corners.
> + // Limit the number of hit-tests and discard failed attempts.
> + int betterX = cursX, betterY = cursY;
> + for (int i=0; i<16; i++) {
> + if ([NSWindow windowNumberAtPoint:NSMakePoint(betterX, hostScreenH -
> betterY)
> + belowWindowWithWindowNumber:0] == self.window.windowNumber) {
> + cursX = betterX;
> + cursY = betterY;
> + break;
> + };
> +
> + if (betterX < scrX + scrW/2) {
> + betterX++;
> + } else {
> + betterX--;
> + }
> +
> + if (betterY < scrY + scrH/2) {
> + betterY++;
> + } else {
> + betterY--;
> + }
> + }
> +
> + // Subtract this warp delta from the next NSEventTypeMouseMoved.
> + // These are in down-is-positive coords, same as NSEvent deltaX/deltaY.
> + warpDeltaX += cursX - lastWarpX;
> + warpDeltaY += cursY - lastWarpY;
> +
> + CGWarpMouseCursorPosition(NSMakePoint(cursX, cursY));
> +
> + lastWarpX = cursX;
> + lastWarpY = cursY;
> +}
> +
> - (BOOL) isMouseGrabbed {return isMouseGrabbed;}
> - (BOOL) isAbsoluteEnabled {return isAbsoluteEnabled;}
> - (float) cdx {return cdx;}
> @@ -1251,6 +1415,7 @@ - (id) init
> error_report("(cocoa) can't create window");
> exit(1);
> }
> + [normalWindow disableCursorRects];
> [normalWindow setAcceptsMouseMovedEvents:YES];
> [normalWindow setTitle:@"QEMU"];
> [normalWindow setContentView:cocoaView];
> @@ -2123,6 +2288,58 @@ static void cocoa_display_init(DisplayState *ds,
> DisplayOptions *opts)
> qemu_clipboard_peer_register(&cbpeer);
> }
>
> +static void cocoa_mouse_set(DisplayChangeListener *dcl, int x, int y, int
> on) {
> + dispatch_async(dispatch_get_main_queue(), ^{
> + [cocoaView mouseSetX:x Y:y on:on];
> + });
> +}
> +
> +// Convert QEMUCursor to NSCursor, then call cursorDefine
> +static void cocoa_cursor_define(DisplayChangeListener *dcl, QEMUCursor
> *cursor) {
> + CFDataRef cfdata = CFDataCreate(
> + /*allocator*/ NULL,
> + /*bytes*/ (void *)cursor->data,
> + /*length*/ sizeof(uint32_t) * cursor->width * cursor->height);
> +
> + CGDataProviderRef dataprovider = CGDataProviderCreateWithCFData(cfdata);
> +
> + CGImageRef cgimage = CGImageCreate(
> + cursor->width, cursor->height,
> + /*bitsPerComponent*/ 8,
> + /*bitsPerPixel*/ 32,
> + /*bytesPerRow*/ sizeof(uint32_t) * cursor->width,
> + /*colorspace*/ CGColorSpaceCreateWithName(kCGColorSpaceSRGB),
> + /*bitmapInfo*/ kCGBitmapByteOrder32Host | kCGImageAlphaLast,
> + /*provider*/ dataprovider,
> + /*decode*/ NULL,
> + /*shouldInterpolate*/ FALSE,
> + /*intent*/ kCGRenderingIntentDefault);
> +
> + NSImage *unscaled = [[NSImage alloc] initWithCGImage:cgimage
> size:NSZeroSize];
> +
> + CFRelease(cfdata);
> + CGDataProviderRelease(dataprovider);
> + CGImageRelease(cgimage);
> +
> + // Nearest-neighbor scale to the possibly "Retina" cursor size
> + NSImage *scaled = [NSImage
> + imageWithSize:NSMakeSize(cursor->width, cursor->height)
> + flipped:NO
> + drawingHandler:^BOOL(NSRect dest) {
> + [NSGraphicsContext currentContext].imageInterpolation =
> NSImageInterpolationNone;
> + [unscaled drawInRect:dest];
> + return YES;
> + }];
> +
> + NSCursor *nscursor = [[NSCursor alloc]
> + initWithImage:scaled
> + hotSpot:NSMakePoint(cursor->hot_x, cursor->hot_y)];
> +
> + dispatch_async(dispatch_get_main_queue(), ^{
> + [cocoaView cursorDefine:nscursor];
> + });
> +}
> +
> static QemuDisplay qemu_display_cocoa = {
> .type = DISPLAY_TYPE_COCOA,
> .init = cocoa_display_init,
