We should also delete the underlying persistent public key...

net/base/keygen_handler_nss.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/base/keygen_handler.h"
 
 #include <pk11pub.h>
 #include <secmod.h>
 #include <ssl.h>
 #include <secder.h>    // DER_Encode()
@@ skipping 211 matching lines @@
   if (privateKey) {
     // On successful keygen we need to keep the private key, of course,
     // or we won't be able to use the client certificate.
     if (!isSuccess || !stores_key_) {
       PK11_DestroyTokenObject(privateKey->pkcs11Slot, privateKey->pkcs11ID);
     }
     SECKEY_DestroyPrivateKey(privateKey);
   }
 
   if (publicKey) {
-    PK11_DestroyTokenObject(publicKey->pkcs11Slot, publicKey->pkcs11ID);
+    if (!isSuccess || !stores_key_) {
+      PK11_DestroyTokenObject(publicKey->pkcs11Slot, publicKey->pkcs11ID);
+    }
+    SECKEY_DestroyPublicKey(publicKey);
   }
   if (spkInfo) {
     SECKEY_DestroySubjectPublicKeyInfo(spkInfo);
   }
-  if (publicKey) {
-    SECKEY_DestroyPublicKey(publicKey);
-  }
   if (arena) {
     PORT_FreeArena(arena, PR_TRUE);
   }
   if (slot != NULL) {
     PK11_FreeSlot(slot);
   }
   if (pkac.challenge.data) {
     free(pkac.challenge.data);
   }
 
   return (isSuccess ? result_blob : std::string());
 }
 
 }  // namespace net