Bug 863947: When using libpkix with both explicit anchors and user/PKCS#11 trust settings, correctly handle cached trust anchors. Patch by Ryan Sleevi. r=rrelyea,wtc

git-svn-id: svn://10.0.0.236/branches/NSS_3_14_4_RELEASE_BRANCH@265072 18797224-902f-48f8-a5cc-f745e15eee43

mozilla/security/nss/lib/libpkix/include/pkix_pl_pki.h

@@ -1488,6 +1488,32 @@ PKIX_PL_Cert_VerifySignature(
         PKIX_PL_PublicKey *pubKey,
         void *plContext);
 
+/* A set of flags to indicate how explicitly configured trust anchors should be
+ * handled by PKIX_PL_Cert_IsCertTrusted
+ */
+typedef enum PKIX_PL_TrustAnchorModeEnum {
+        /* Indicates trust anchors should be ignored; only the underlying
+         * platform's trust settings should be used.
+         */
+        PKIX_PL_TrustAnchorMode_Ignore,
+
+        /* Indicates that explicitly configured trust anchors may be considered
+         * trustworthy, if present.
+         * Note: If the underlying platform supports marking a certificate as
+         *       explicitly untrustworthy, explicitly configured trust anchors
+         *       MAY be ignored/rejected.
+         */
+        PKIX_PL_TrustAnchorMode_Additive,
+
+        /* Indicates that ONLY trust anchors should be considered as
+         * trustworthy.
+         * Note: If the underlying platform supports marking a certificate as
+         *       explicitly untrustworthy, explicitly configured trust anchors
+         *       MAY be ignored/rejected.
+         */
+        PKIX_PL_TrustAnchorMode_Exclusive,
+} PKIX_PL_TrustAnchorMode;
+
 /*
  * FUNCTION: PKIX_PL_Cert_IsCertTrusted
  * DESCRIPTION:
@@ -1509,8 +1535,9 @@ PKIX_PL_Cert_VerifySignature(
  *  "cert"
  *      Address of Cert whose trustworthiness is to be determined. Must be
  *      non-NULL.
- *  "trustOnlyUserAnchors"
+ *  "trustAnchorMode"
- *      States that we can only trust explicitly defined user trust anchors.
+ *      A PKIX_PL_TrustAnchorMode that indicates how explicitly defined user
+ *      trust anchors should be handled.
  *  "pTrusted"
  *      Address where the Boolean value will be stored. Must be non-NULL.
  *  "plContext"
@@ -1525,7 +1552,7 @@ PKIX_PL_Cert_VerifySignature(
 PKIX_Error *
 PKIX_PL_Cert_IsCertTrusted(
         PKIX_PL_Cert *cert,
-        PKIX_Boolean trustOnlyUserAnchors,
+        PKIX_PL_TrustAnchorMode trustAnchorMode,
         PKIX_Boolean *pTrusted,
         void *plContext);
 

mozilla/security/nss/lib/libpkix/pkix/top/pkix_build.c

@@ -836,7 +836,8 @@ pkix_Build_VerifyCertificate(
         PKIX_PL_PublicKey *candidatePubKey = NULL;
         PKIX_CertChainChecker *userChecker = NULL;
         PKIX_CertChainChecker_CheckCallback checkerCheck = NULL;
-        PKIX_Boolean trustOnlyUserAnchors = PKIX_FALSE;
+        PKIX_PL_TrustAnchorMode trustAnchorMode =
+                PKIX_PL_TrustAnchorMode_Ignore;
         void *nbioContext = NULL;
         
         PKIX_ENTER(BUILD, "pkix_Build_VerifyCertificate");
@@ -850,12 +851,17 @@ pkix_Build_VerifyCertificate(
         candidateCert = state->candidateCert;
 
         if (state->buildConstants.numAnchors) {
-            trustOnlyUserAnchors = state->buildConstants.trustOnlyUserAnchors;
+            if (state->buildConstants.trustOnlyUserAnchors) {
+                trustAnchorMode = PKIX_PL_TrustAnchorMode_Exclusive;
+            } else {
+                trustAnchorMode = PKIX_PL_TrustAnchorMode_Additive;
+            }
+        } else {
+            trustAnchorMode = PKIX_PL_TrustAnchorMode_Ignore;
         }
 
         PKIX_CHECK(
-            PKIX_PL_Cert_IsCertTrusted(candidateCert,
+            PKIX_PL_Cert_IsCertTrusted(candidateCert, trustAnchorMode,
-                                       trustOnlyUserAnchors,
                                        &trusted, plContext),
             PKIX_CERTISCERTTRUSTEDFAILED);
 
@@ -3041,19 +3047,27 @@ pkix_Build_CheckInCache(
                    (matchingAnchor, &trustedCert, plContext),
                    PKIX_TRUSTANCHORGETTRUSTEDCERTFAILED);
         
-        if (state->buildConstants.anchors &&
+        if (anchors && state->buildConstants.numAnchors) {
-            state->buildConstants.anchors->length) {
             /* Check if it is one of the trust anchors */
             PKIX_CHECK(
-                pkix_List_Contains(state->buildConstants.anchors,
+                pkix_List_Contains(anchors,
                                    (PKIX_PL_Object *)matchingAnchor,
                                    &trusted,
                                    plContext),
                 PKIX_LISTCONTAINSFAILED);
-        } else {
+        }
-            PKIX_CHECK(PKIX_PL_Cert_IsCertTrusted
+
-                       (trustedCert, PKIX_FALSE, &trusted, plContext),
+        if ((!trusted && !state->buildConstants.trustOnlyUserAnchors) ||
-                       PKIX_CERTISCERTTRUSTEDFAILED);
+            !state->buildConstants.numAnchors) {
+            /* If it is not one of the trust anchors and the trust anchors
+             * are supplemental, or if there are no trust anchors, then check
+             * if the cert is trusted directly.
+             */
+            PKIX_CHECK(
+                PKIX_PL_Cert_IsCertTrusted(trustedCert,
+                                           PKIX_PL_TrustAnchorMode_Ignore,
+                                           &trusted, plContext),
+                PKIX_CERTISCERTTRUSTEDFAILED);
         }
 
         if (!trusted) {

mozilla/security/nss/lib/libpkix/pkix_pl_nss/pki/pkix_pl_cert.c

@@ -3294,7 +3294,7 @@ pkix_pl_Cert_GetTrusted(void *plContext,
 PKIX_Error *
 PKIX_PL_Cert_IsCertTrusted(
         PKIX_PL_Cert *cert,
-        PKIX_Boolean trustOnlyUserAnchors,
+        PKIX_PL_TrustAnchorMode trustAnchorMode,
         PKIX_Boolean *pTrusted,
         void *plContext)
 {
@@ -3315,8 +3315,10 @@ PKIX_PL_Cert_IsCertTrusted(
                 PKIX_ERROR(PKIX_CERTISCERTTRUSTEDFAILED);
         }
 
-        if (trustOnlyUserAnchors || cert->isUserTrustAnchor) {
+        if (trustAnchorMode == PKIX_PL_TrustAnchorMode_Exclusive ||
-            /* discard our |trusted| value since we are using the anchors */
+            (trustAnchorMode == PKIX_PL_TrustAnchorMode_Additive &&
+             cert->isUserTrustAnchor)) {
+            /* Use the trust anchor's |trusted| value */
             *pTrusted = cert->isUserTrustAnchor;
             goto cleanup;
         }