diff --git a/include/nanogui/screen.h b/include/nanogui/screen.h
index 709789b4da..0638b3e45b 100644
--- a/include/nanogui/screen.h
+++ b/include/nanogui/screen.h
@@ -195,7 +195,7 @@ class NANOGUI_EXPORT Screen : public Widget {
     int mMouseState, mModifiers;
     Vector2i mMousePos;
     bool mDragActive;
-    Widget *mDragWidget = nullptr;
+    ref<Widget> mDragWidget;
     double mLastInteraction;
     bool mProcessEvents;
     Color mBackground;
diff --git a/src/screen.cpp b/src/screen.cpp
index 2a2634f07a..93b190336d 100644
--- a/src/screen.cpp
+++ b/src/screen.cpp
@@ -480,6 +480,12 @@ bool Screen::cursorPosCallbackEvent(double x, double y) {
     try {
         p -= Vector2i(1, 2);
 
+        // Make sure mDragWidget isn't the only remaining reference
+        if (mDragActive && mDragWidget->getRefCount() == 1) {
+            mDragActive = false;
+            mDragWidget = nullptr;
+        }
+
         if (!mDragActive) {
             Widget *widget = findWidget(p);
             if (widget != nullptr && widget->cursor() != mCursor) {
@@ -522,6 +528,12 @@ bool Screen::mouseButtonCallbackEvent(int button, int action, int modifiers) {
         else
             mMouseState &= ~(1 << button);
 
+        // Make sure mDragWidget isn't the only remaining reference
+        if (mDragActive && mDragWidget->getRefCount() == 1) {
+            mDragActive = false;
+            mDragWidget = nullptr;
+        }
+
         auto dropWidget = findWidget(mMousePos);
         if (mDragActive && action == GLFW_RELEASE &&
             dropWidget != mDragWidget)
@@ -536,7 +548,7 @@ bool Screen::mouseButtonCallbackEvent(int button, int action, int modifiers) {
 
         if (action == GLFW_PRESS && (button == GLFW_MOUSE_BUTTON_1 || button == GLFW_MOUSE_BUTTON_2)) {
             mDragWidget = findWidget(mMousePos);
-            if (mDragWidget == this)
+            if (mDragWidget.get() == this)
                 mDragWidget = nullptr;
             mDragActive = mDragWidget != nullptr;
             if (!mDragActive)
@@ -649,7 +661,7 @@ void Screen::updateFocus(Widget *widget) {
 void Screen::disposeWindow(Window *window) {
     if (std::find(mFocusPath.begin(), mFocusPath.end(), window) != mFocusPath.end())
         mFocusPath.clear();
-    if (mDragWidget == window)
+    if (mDragWidget.get() == window)
         mDragWidget = nullptr;
     removeChild(window);
 }
diff --git a/src/widget.cpp b/src/widget.cpp
index 90b2461818..db81808ad4 100644
--- a/src/widget.cpp
+++ b/src/widget.cpp
@@ -152,12 +152,19 @@ void Widget::addChild(Widget * widget) {
 }
 
 void Widget::removeChild(const Widget *widget) {
-    mChildren.erase(std::remove(mChildren.begin(), mChildren.end(), widget), mChildren.end());
-    widget->decRef();
+    auto it = std::find(mChildren.begin(), mChildren.end(), widget);
+    if (it == mChildren.end()) {
+        return;
+    }
+    removeChild(it - mChildren.begin());
 }
 
 void Widget::removeChild(int index) {
     Widget *widget = mChildren[index];
+    if (widget->focused()) {
+        requestFocus();
+    }
+
     mChildren.erase(mChildren.begin() + index);
     widget->decRef();
 }