diff --git a/bufferevent_sock.c b/bufferevent_sock.c
index ab35f879..58095f37 100644
--- a/bufferevent_sock.c
+++ b/bufferevent_sock.c
@@ -192,6 +192,10 @@ bufferevent_readcb(evutil_socket_t fd, short event, void *arg)
 		int err = evutil_socket_geterror(fd);
 		if (EVUTIL_ERR_RW_RETRIABLE(err))
 			goto reschedule;
+		if (EVUTIL_ERR_CONNECT_REFUSED(err)) {
+			bufev_p->connection_refused = 1;
+			goto done;
+		}
 		/* error case */
 		what |= BEV_EVENT_ERROR;
 	} else if (res == 0) {