crypto: disable NSS AES-NI support when AVX is disabled by OS.

crypto/nss_util.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "crypto/nss_util.h"
 #include "crypto/nss_util_internal.h"
 
 #include <nss.h>
 #include <pk11pub.h>
 #include <plarena.h>
 #include <prerror.h>
 #include <prinit.h>
 #include <prtime.h>
 #include <secmod.h>
+#include <stdlib.h>
 
 #if defined(OS_LINUX)
 #include <linux/nfs_fs.h>
 #include <sys/vfs.h>
 #elif defined(OS_OPENBSD)
 #include <sys/mount.h>
 #include <sys/param.h>
 #endif
 
 #include <vector>
 
+#include "base/cpu.h"
 #include "base/debug/alias.h"
 #include "base/debug/stack_trace.h"
 #include "base/environment.h"
 #include "base/file_util.h"
 #include "base/files/file_path.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/metrics/histogram.h"
@@ skipping 389 matching lines @@
         test_slot_(NULL),
         tpm_slot_(NULL),
         root_(NULL),
         chromeos_user_logged_in_(false) {
     base::TimeTicks start_time = base::TimeTicks::Now();
 
     // It's safe to construct on any thread, since LazyInstance will prevent any
     // other threads from accessing until the constructor is done.
     thread_checker_.DetachFromThread();
 
+    DisableAESNIIfNeeded();
+
     EnsureNSPRInit();
 
     // We *must* have NSS >= 3.14.3.
     COMPILE_ASSERT(
         (NSS_VMAJOR == 3 && NSS_VMINOR == 14 && NSS_VPATCH >= 3) ||
         (NSS_VMAJOR == 3 && NSS_VMINOR > 14) ||
         (NSS_VMAJOR > 3),
         nss_version_check_failed);
     // Also check the run-time NSS version.
     // NSS_VersionCheck is a >= check, not strict equality.
@@ skipping 172 matching lines @@
       if (PK11_NeedUserInit(db_slot))
         PK11_InitPin(db_slot, NULL, NULL);
     }
     else {
       LOG(ERROR) << "Error opening persistent database (" << modspec
                  << "): " << GetNSSErrorMessage();
     }
     return db_slot;
   }
 
+  static void DisableAESNIIfNeeded() {
+    base::CPU cpu;
+
+    if (cpu.has_avx_hardware() && !cpu.has_avx()) {
+      // Some versions of NSS have a bug that causes AVX instructions to be
+      // used without testing whether XSAVE is enabled by the operating system.
+      // In order to work around this, we disable AES-NI in NSS when we find
+      // that |has_avx()| is false (which includes the XSAVE test).

[Ryan Sleevi, 2013/11/20 21:46:28]

Let's include an NSS version check (or softoken version check) here so that
we're not always disabling it on Linux.

The softoken version is exposed via PKCS#11 as part of the library version
.major/.minor, which will handle the cases (such as RHEL) where softoken may be
older than NSS.

bool use_hw_aes_if_present = true;
CK_TOKEN_INFO token_info;
ScopedPK11Slot slot(PK11_GetInternalSlot());
if (!slot.get() || PK11_GetTokenInfo(slot, &token_info) != SECSuccess) {
  use_hw_aes_if_present = false;
}
if (token_info.hardwareVersion.major < XXX ||
    (token_info.hardwareVersion.major == XXX &&
     token_info.hardwareVersion.minor < YYY)) {
  use_hw_aes_if_present = cpu.has_avx_hardware() && cpu.has_avx();
}
if (!use_hw_aes_if_present) {
  setenv(...)
}

[wtc, 2013/11/21 21:03:43]

In this particular case, it is enough to check the version of the main NSS
library.

We can safely assume this bug will be fixed in NSS 3.15.4. Since Softoken 3.15.3
is not FIPS 140-2 validated, there is no reason for RHEL to use a newer version
of NSS with Softoken 3.15.3.

We can use the test
    NSS_VersionCheck("3.15") && !NSS_VersionCheck("3.15.4")
to detect the buggy version range (3.15 <= version < 3.15.4). Although the bug
was introduced in NSS 3.14.2, there were no NSS PK11_ functions to access
AES-GCM until NSS 3.15.

+      setenv("NSS_DISABLE_HW_AES", "1", 1 /* overwrite */);
+    }
+  }
+
   // If this is set to true NSS is forced to be initialized without a DB.
   static bool force_nodb_init_;
 
   bool tpm_token_enabled_for_nss_;
   std::string tpm_token_name_;
   std::string tpm_user_pin_;
   SECMODModule* chaps_module_;
   PK11SlotInfo* software_slot_;
   PK11SlotInfo* test_slot_;
   PK11SlotInfo* tpm_slot_;
@@ skipping 189 matching lines @@
 
 PK11SlotInfo* GetPublicNSSKeySlot() {
   return g_nss_singleton.Get().GetPublicNSSKeySlot();
 }
 
 PK11SlotInfo* GetPrivateNSSKeySlot() {
   return g_nss_singleton.Get().GetPrivateNSSKeySlot();
 }
 
 }  // namespace crypto