(nuttx) branch master updated: pty: use mutex to protect alloc minor

[xiaoxiang]

This is an automated email from the ASF dual-hosted git repository.

xiaoxiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nuttx.git


The following commit(s) were added to refs/heads/master by this push:
     new 6a38c37702 pty: use mutex to protect alloc minor
6a38c37702 is described below

commit 6a38c377025f392a0ae02f480314d48b05705ecd
Author: yangsong8 <yangso...@xiaomi.com>
AuthorDate: Fri Aug 9 11:22:38 2024 +0800

    pty: use mutex to protect alloc minor
    
    If failed in pty_register2, it is possible to enter unregister_driver
    function, which eventually calls ptmx_minor_free, resulting in mutex
    conflict.
    
    Signed-off-by: yangsong8 <yangso...@xiaomi.com>
---
 drivers/serial/ptmx.c | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/drivers/serial/ptmx.c b/drivers/serial/ptmx.c
index 39f311f65c..467afe09d0 100644
--- a/drivers/serial/ptmx.c
+++ b/drivers/serial/ptmx.c
@@ -180,10 +180,10 @@ static int ptmx_open(FAR struct file *filep)
   /* Allocate a PTY minor */
 
   minor = ptmx_minor_allocate();
+  nxmutex_unlock(&g_ptmx.px_lock);
   if (minor < 0)
     {
-      ret = minor;
-      goto errout_with_lock;
+      return minor;
     }
 
   /* Create the master slave pair.  This should create:
@@ -197,7 +197,8 @@ static int ptmx_open(FAR struct file *filep)
   ret = pty_register2(minor, true);
   if (ret < 0)
     {
-      goto errout_with_minor;
+      ptmx_minor_free(minor);
+      return ret;
     }
 
   /* Open the master device:  /dev/ptyN, where N=minor */
@@ -220,15 +221,7 @@ static int ptmx_open(FAR struct file *filep)
   ret = unregister_driver(devname);
   DEBUGASSERT(ret >= 0 || ret == -EBUSY);  /* unregister_driver() should never 
fail */
 
-  nxmutex_unlock(&g_ptmx.px_lock);
   return OK;
-
-errout_with_minor:
-  ptmx_minor_free(minor);
-
-errout_with_lock:
-  nxmutex_unlock(&g_ptmx.px_lock);
-  return ret;
 }
 
 /****************************************************************************