diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 00a90bb..4eb4b5a 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -323,9 +323,14 @@
     }
   }
 
+  bool added_port = false;
   for (auto port = ports_.begin(); port != ports_.end(); port++) {
     int r = server->AddListeningPort(port->addr, port->creds.get());
-    if (!r) return nullptr;
+    if (!r) {
+      if (added_port) server->Shutdown();
+      return nullptr;
+    }
+    added_port = true;
     if (port->selected_port != nullptr) {
       *port->selected_port = r;
     }
@@ -333,6 +338,7 @@
 
   auto cqs_data = cqs_.empty() ? nullptr : &cqs_[0];
   if (!server->Start(cqs_data, cqs_.size())) {
+    if (added_port) server->Shutdown();
     return nullptr;
   }