diff --git a/src/server.c b/src/server.c
index b0e0ba0c4..c076e0cc1 100644
--- a/src/server.c
+++ b/src/server.c
@@ -701,11 +701,6 @@ server_recv_cb(EV_P_ ev_io *w, int revents)
 
     buffer_t *buf = server->buf;
 
-    // Ignore any new packet if the server is stopped
-    if (server->stage == STAGE_STOP) {
-        return;
-    }
-
     if (server->stage == STAGE_STREAM) {
         remote = server->remote;
         buf    = remote->buf;
@@ -731,6 +726,11 @@ server_recv_cb(EV_P_ ev_io *w, int revents)
         }
     }
 
+    // Ignore any new packet if the server is stopped
+    if (server->stage == STAGE_STOP) {
+        return;
+    }
+
     tx      += r;
     buf->len = r;
 
@@ -1101,11 +1101,6 @@ remote_recv_cb(EV_P_ ev_io *w, int revents)
     remote_t *remote              = remote_recv_ctx->remote;
     server_t *server              = remote->server;
 
-    // Ignore any new packet if the server is stopped
-    if (server->stage == STAGE_STOP) {
-        return;
-    }
-
     if (server == NULL) {
         LOGE("invalid server");
         close_and_free_remote(EV_A_ remote);
@@ -1134,6 +1129,11 @@ remote_recv_cb(EV_P_ ev_io *w, int revents)
 
     rx += r;
 
+    // Ignore any new packet if the server is stopped
+    if (server->stage == STAGE_STOP) {
+        return;
+    }
+
     server->buf->len = r;
     int err = crypto->encrypt(server->buf, server->e_ctx, SOCKET_BUF_SIZE);