diff --git a/ui/vnc.c b/ui/vnc.c
index eb1fcb3..a8cce5a 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -731,6 +731,10 @@
     qemu_pixman_image_unref(vd->server);
     vd->server = NULL;
 
+    if (QTAILQ_EMPTY(&vd->clients)) {
+        return;
+    }
+
     vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
                                           vnc_width(vd),
                                           vnc_height(vd),
@@ -1241,6 +1245,10 @@
     if (vs->initialized) {
         QTAILQ_REMOVE(&vs->vd->clients, vs, next);
         qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
+        if (QTAILQ_EMPTY(&vs->vd->clients)) {
+            /* last client gone */
+            vnc_update_server_surface(vs->vd);
+        }
     }
 
     if (vs->vd->lock_key_sync)
@@ -3079,6 +3087,7 @@
 {
     vs->initialized = true;
     VncDisplay *vd = vs->vd;
+    bool first_client = QTAILQ_EMPTY(&vd->clients);
 
     vs->last_x = -1;
     vs->last_y = -1;
@@ -3092,6 +3101,9 @@
     vs->bh = qemu_bh_new(vnc_jobs_bh, vs);
 
     QTAILQ_INSERT_TAIL(&vd->clients, vs, next);
+    if (first_client) {
+        vnc_update_server_surface(vd);
+    }
 
     graphic_hw_update(vd->dcl.con);