[PATCH 2/2] drm/vmwgfx: Filter out modes those cannot be supported by the current VRAM size.

[Thomas Hellstrom]

From: Sinclair Yeh <s...@vmware.com>

When screen objects are enabled, the bpp is assumed to be 32, otherwise
it is set to 16.

v2:
* Use u32 instead of u64 for assumed_bpp.
* Fixed mechanism to check for screen objects
* Limit the back buffer size to VRAM.

Signed-off-by: Sinclair Yeh <syeh at vmware.com>
Reviewed-by: Thomas Hellstrom <thellstrom at vmware.com>
Cc: <stable at vger.kernel.org>
---
 drivers/gpu/drm/vmwgfx/vmwgfx_drv.c |  6 +++++-
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.c | 16 +++++++++++++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c 
b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 7197af1..25f3c25 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -688,7 +688,11 @@ static int vmw_driver_load(struct drm_device *dev, 
unsigned long chipset)
                goto out_err0;
        }

-       if (unlikely(dev_priv->prim_bb_mem < dev_priv->vram_size))
+       /*
+        * Limit back buffer size to VRAM size.  Remove this once
+        * screen targets are implemented.
+        */
+       if (dev_priv->prim_bb_mem > dev_priv->vram_size)
                dev_priv->prim_bb_mem = dev_priv->vram_size;

        mutex_unlock(&dev_priv->hw_mutex);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c 
b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 8fc1e38..941a7bc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1950,6 +1950,14 @@ int vmw_du_connector_fill_modes(struct drm_connector 
*connector,
                DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC)
        };
        int i;
+       u32 assumed_bpp = 2;
+
+       /*
+        * If using screen objects, then assume 32-bpp because that's what the
+        * SVGA device is assuming
+        */
+       if (dev_priv->sou_priv)
+               assumed_bpp = 4;

        /* Add preferred mode */
        {
@@ -1960,8 +1968,9 @@ int vmw_du_connector_fill_modes(struct drm_connector 
*connector,
                mode->vdisplay = du->pref_height;
                vmw_guess_mode_timing(mode);

-               if (vmw_kms_validate_mode_vram(dev_priv, mode->hdisplay * 2,
-                                              mode->vdisplay)) {
+               if (vmw_kms_validate_mode_vram(dev_priv,
+                                               mode->hdisplay * assumed_bpp,
+                                               mode->vdisplay)) {
                        drm_mode_probed_add(connector, mode);
                } else {
                        drm_mode_destroy(dev, mode);
@@ -1983,7 +1992,8 @@ int vmw_du_connector_fill_modes(struct drm_connector 
*connector,
                    bmode->vdisplay > max_height)
                        continue;

-               if (!vmw_kms_validate_mode_vram(dev_priv, bmode->hdisplay * 2,
+               if (!vmw_kms_validate_mode_vram(dev_priv,
+                                               bmode->hdisplay * assumed_bpp,
                                                bmode->vdisplay))
                        continue;

-- 
1.8.3.2