Bug#595386: xserver-xorg: X segfaults on switching from text console with kms (infinite recursion in xf86RandR12EnterVT())

[Julien Cristau]

On Fri, Sep  3, 2010 at 20:08:55 +0300, Sami Liedes wrote:

> I can cause X to segfault by doing the following after a fresh reboot:
> 
> PROCEDURE:
> 
> 1. log in to virtual consoles 1 and 2
> [in virtual console 1]
> 2. sudo /etc/init.d/kdm stop
> 3. X
> [console is automatically switched to X]
> 4. go to vc 2 by pressing ctrl-alt-f2
> 5. DISPLAY=:0 startkde
> 6. switch to the vc where X is running (ctrl-alt-f8)
> 7. switch back to vc 1 (ctrl-alt-f1) while kde is starting (KDE's
>    splash screen is visible)
> 8. wait for a few seconds
> 9. switch back to X's vc (ctrl-alt-f8)
> 
> EXPECTED OUTCOME:
> 
> 10. I now have a newly started KDE session on my screen
> 
> ACTUAL OUTCOME:
> 
> 10. Screen hangs. Text console is still visible, but the cursor has
> stopped blinking. On further inspection, X segfaults.
> 
> Starting X (remotely, via ssh) restores the console to an usable
> state. Other than that, there seems to be no way to recover (isn't kms
> supposed to fix things like that?).
> 
Could be fixed by:

commit 68a9ee8370e6f9b38218376ac92d5130a5b0ef1e
Author: Keith Packard <kei...@keithp.com>
Date:   Fri May 7 22:56:04 2010 -0700

    Clean up RandR12 bits on screen close (bug 27114)
    
    When resetting the server, pScrn->EnterVT must be unwrapped or the
    next server generation will end up wrapping the wrapper and causing an
    infinite recursion on EnterVT.
    
    Signed-off-by: Keith Packard <kei...@keithp.com>
    Tested-by: Michael Stapelberg <michael+freedesk...@stapelberg.de>

Can you try that (attached)?

Cheers,
Julien

From 68a9ee8370e6f9b38218376ac92d5130a5b0ef1e Mon Sep 17 00:00:00 2001
From: Keith Packard <kei...@keithp.com>
Date: Fri, 7 May 2010 22:56:04 -0700
Subject: [PATCH] Clean up RandR12 bits on screen close (bug 27114)

When resetting the server, pScrn->EnterVT must be unwrapped or the
next server generation will end up wrapping the wrapper and causing an
infinite recursion on EnterVT.

Signed-off-by: Keith Packard <kei...@keithp.com>
Tested-by: Michael Stapelberg <michael+freedesk...@stapelberg.de>
---
 hw/xfree86/modes/xf86Crtc.c    |    2 ++
 hw/xfree86/modes/xf86RandR12.c |   26 +++++++++++++++++++++++++-
 hw/xfree86/modes/xf86RandR12.h |    1 +
 3 files changed, 28 insertions(+), 1 deletions(-)

diff --git a/hw/xfree86/modes/xf86Crtc.c b/hw/xfree86/modes/xf86Crtc.c
index a62a63f..7f6fad3 100644
--- a/hw/xfree86/modes/xf86Crtc.c
+++ b/hw/xfree86/modes/xf86Crtc.c
@@ -753,6 +753,8 @@ xf86CrtcCloseScreen (int index, ScreenPtr screen)
 
 	crtc->randr_crtc = NULL;
     }
+    xf86RandR12CloseScreen (screen);
+
     return screen->CloseScreen (index, screen);
 }
 
diff --git a/hw/xfree86/modes/xf86RandR12.c b/hw/xfree86/modes/xf86RandR12.c
index bb4d75e..043ceee 100644
--- a/hw/xfree86/modes/xf86RandR12.c
+++ b/hw/xfree86/modes/xf86RandR12.c
@@ -928,6 +928,24 @@ xf86RandR12Init (ScreenPtr pScreen)
 }
 
 void
+xf86RandR12CloseScreen (ScreenPtr pScreen)
+{
+    XF86RandRInfoPtr	randrp;
+
+#if XORG_VERSION_CURRENT < XORG_VERSION_NUMERIC(7,0,0,0,0)
+    if (xf86RandR12Key == NULL)
+	return;
+#endif
+
+    randrp = XF86RANDRINFO(pScreen);
+#if RANDR_12_INTERFACE
+    xf86Screens[pScreen->myNum]->EnterVT = randrp->orig_EnterVT;
+#endif
+
+    free(randrp);
+}
+
+void
 xf86RandR12SetRotations (ScreenPtr pScreen, Rotation rotations)
 {
     XF86RandRInfoPtr	randrp;
@@ -1755,10 +1773,16 @@ static Bool
 xf86RandR12EnterVT (int screen_index, int flags)
 {
     ScreenPtr        pScreen = screenInfo.screens[screen_index];
+    ScrnInfoPtr	     pScrn = xf86Screens[screen_index];
     XF86RandRInfoPtr randrp  = XF86RANDRINFO(pScreen);
+    Bool	     ret;
 
     if (randrp->orig_EnterVT) {
-	if (!randrp->orig_EnterVT (screen_index, flags))
+	pScrn->EnterVT = randrp->orig_EnterVT;
+	ret = pScrn->EnterVT (screen_index, flags);
+	randrp->orig_EnterVT = pScrn->EnterVT;
+	pScrn->EnterVT = xf86RandR12EnterVT;
+	if (!ret)
 	    return FALSE;
     }
 
diff --git a/hw/xfree86/modes/xf86RandR12.h b/hw/xfree86/modes/xf86RandR12.h
index c8d9918..0c586be 100644
--- a/hw/xfree86/modes/xf86RandR12.h
+++ b/hw/xfree86/modes/xf86RandR12.h
@@ -30,6 +30,7 @@
 
 extern _X_EXPORT Bool xf86RandR12CreateScreenResources (ScreenPtr pScreen);
 extern _X_EXPORT Bool xf86RandR12Init(ScreenPtr pScreen);
+extern _X_EXPORT void xf86RandR12CloseScreen(ScreenPtr pScreen);
 extern _X_EXPORT void xf86RandR12SetRotations (ScreenPtr pScreen, Rotation rotation);
 extern _X_EXPORT void xf86RandR12SetTransformSupport (ScreenPtr pScreen, Bool transforms);
 extern _X_EXPORT Bool xf86RandR12SetConfig(ScreenPtr pScreen, Rotation rotation, int rate,
-- 
1.7.1

signature.asc
Description: Digital signature