diff --git a/testgres/backup.py b/testgres/backup.py
index 388697b..857c46d 100644
--- a/testgres/backup.py
+++ b/testgres/backup.py
@@ -184,14 +184,19 @@ def spawn_replica(self, name=None, destroy=True, slot=None):
         """
 
         # Build a new PostgresNode
-        with clean_on_error(self.spawn_primary(name=name,
-                                               destroy=destroy)) as node:
+        node = self.spawn_primary(name=name, destroy=destroy)
+        assert node is not None
 
+        try:
             # Assign it a master and a recovery file (private magic)
             node._assign_master(self.original_node)
             node._create_recovery_conf(username=self.username, slot=slot)
+        except:  # noqa: E722
+            # TODO: Pass 'final=True' ?
+            node.cleanup(release_resources=True)
+            raise
 
-            return node
+        return node
 
     def cleanup(self):
         """
diff --git a/testgres/node.py b/testgres/node.py
index 41504e8..defc0b4 100644
--- a/testgres/node.py
+++ b/testgres/node.py
@@ -231,8 +231,6 @@ def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
-        self.free_port()
-
         # NOTE: Ctrl+C does not count!
         got_exception = type is not None and type != KeyboardInterrupt
 
@@ -246,6 +244,8 @@ def __exit__(self, type, value, traceback):
         else:
             self._try_shutdown(attempts)
 
+        self._release_resources()
+
     def __repr__(self):
         return "{}(name='{}', port={}, base_dir='{}')".format(
             self.__class__.__name__,
@@ -663,6 +663,9 @@ def _try_shutdown(self, max_attempts, with_force=False):
             ps_output,
             ps_command)
 
+    def _release_resources(self):
+        self.free_port()
+
     @staticmethod
     def _throw_bugcheck__unexpected_result_of_ps(result, cmd):
         assert type(result) == str  # noqa: E721
@@ -1340,7 +1343,7 @@ def free_port(self):
             self._port = None
             self._port_manager.release_port(port)
 
-    def cleanup(self, max_attempts=3, full=False):
+    def cleanup(self, max_attempts=3, full=False, release_resources=False):
         """
         Stop node if needed and remove its data/logs directory.
         NOTE: take a look at TestgresConfig.node_cleanup_full.
@@ -1363,6 +1366,9 @@ def cleanup(self, max_attempts=3, full=False):
 
         self.os_ops.rmdirs(rm_dir, ignore_errors=False)
 
+        if release_resources:
+            self._release_resources()
+
         return self
 
     @method_decorator(positional_args_hack(['dbname', 'query']))