diff --git a/libp2p/switch.nim b/libp2p/switch.nim
index d6ea195cb9..bc225e6966 100644
--- a/libp2p/switch.nim
+++ b/libp2p/switch.nim
@@ -223,6 +223,11 @@ proc upgradeMonitor(
   finally:
     upgrades.release()
 
+proc acceptCleanup(upgrades: AsyncSemaphore, connection: Connection) {.async.} =
+  upgrades.release() # always release the slot
+  if not isNil(connection):
+    await connection.close()
+
 proc accept(s: Switch, transport: Transport) {.async.} = # noraises
   ## switch accept loop, ran for every transport
   ##
@@ -263,11 +268,13 @@ proc accept(s: Switch, transport: Transport) {.async.} = # noraises
       trace "releasing semaphore on cancellation"
       upgrades.release() # always release the slot
       return
+    except TransportUseClosedError:
+      trace "Graceful shutdown in accept loop, exiting"
+      await acceptCleanup(upgrades, conn)
+      return
     except CatchableError as exc:
       error "Exception in accept loop, exiting", exc = exc.msg
-      upgrades.release() # always release the slot
-      if not isNil(conn):
-        await conn.close()
+      await acceptCleanup(upgrades, conn)
       return
 
 proc stop*(s: Switch) {.async, public.} =