diff --git a/squeaknode/network/connection.py b/squeaknode/network/connection.py
index 6659f1d5..3dad8618 100644
--- a/squeaknode/network/connection.py
+++ b/squeaknode/network/connection.py
@@ -45,11 +45,11 @@ class Connection(object):
 
     @contextmanager
     def connect(self, connection_manager):
-        self.start_receiving_msgs()
-        self.handshake()
-        logger.debug("Adding peer.")
-        connection_manager.add_peer(self.peer)
         try:
+            self.start_receiving_msgs()
+            self.handshake()
+            logger.debug("Adding peer.")
+            connection_manager.add_peer(self.peer)
             logger.debug("Yielding peer.")
             yield self
         except Exception:
diff --git a/squeaknode/network/connection_manager.py b/squeaknode/network/connection_manager.py
index 1a41407c..feaa11f2 100644
--- a/squeaknode/network/connection_manager.py
+++ b/squeaknode/network/connection_manager.py
@@ -94,8 +94,7 @@ class ConnectionManager(object):
         """
         with self.peers_lock:
             if not self.has_connection(peer.remote_address):
-                logger.debug('Failed to remove peer {}'.format(peer))
-                raise MissingPeerError()
+                return
             del self._peers[peer.remote_address]
             logger.debug('Removed peer {}'.format(peer))
             self.on_peers_changed()