Reject PSS salt lengths that do not fit in a u8

`get_pss_signature_algo_id` cast the salt length to `u8` with `as`,
silently truncating any value >= 256 (e.g. 256 -> 0). The resulting
RSASSA-PSS `AlgorithmIdentifier` then advertised the wrong salt length,
so signatures produced with such a salt length failed verification.

Take the salt length as `usize` in the helper and convert it with
`u8::try_from`, returning a DER `Overflow` error when it does not fit,
rather than mis-encoding it. Both `SigningKey` and `BlindedSigningKey`
funnel through this helper, so both signing paths are covered.

Fixes #703.

Author: cong-or

src/pss.rs

@@ -39,7 +39,10 @@ use {
     crate::encoding::ID_RSASSA_PSS,
     const_oid::AssociatedOid,
     pkcs1::RsaPssParams,
-    spki::{der::Any, AlgorithmIdentifierOwned},
+    spki::{
+        der::{Any, ErrorKind},
+        AlgorithmIdentifierOwned,
+    },
 };
 
 /// Digital signatures using PSS padding.
@@ -272,15 +275,18 @@ pub fn get_default_pss_signature_algo_id<D>() -> spki::Result<AlgorithmIdentifie
 where
     D: Digest + AssociatedOid,
 {
-    let salt_len: u8 = <D as Digest>::output_size() as u8;
-    get_pss_signature_algo_id::<D>(salt_len)
+    get_pss_signature_algo_id::<D>(<D as Digest>::output_size())
 }
 
 #[cfg(feature = "encoding")]
-fn get_pss_signature_algo_id<D>(salt_len: u8) -> spki::Result<AlgorithmIdentifierOwned>
+fn get_pss_signature_algo_id<D>(salt_len: usize) -> spki::Result<AlgorithmIdentifierOwned>
 where
     D: Digest + AssociatedOid,
 {
+    // The salt length is encoded as a single byte in `RsaPssParams`, so reject values that
+    // would not round-trip rather than silently truncating them (which produced an
+    // `AlgorithmIdentifier` advertising the wrong salt length and broke verification). See #703.
+    let salt_len = u8::try_from(salt_len).map_err(|_| ErrorKind::Overflow.to_error())?;
     let pss_params = RsaPssParams::new::<D>(salt_len);
 
     Ok(AlgorithmIdentifierOwned {
@@ -672,4 +678,27 @@ tAboUGBxTDq3ZroNism3DaMIbKPyYrAqhKov1h5V
                 .expect("verification to succeed");
         }
     }
+
+    // Regression test for #703: a salt length that does not fit in a `u8` was silently truncated
+    // when building the PSS `AlgorithmIdentifier` (e.g. 256 wrapped to 0), producing an identifier
+    // that advertised the wrong salt length and broke signature verification. Such lengths must
+    // now be rejected instead of mis-encoded.
+    #[test]
+    fn signature_algorithm_identifier_rejects_oversized_salt_len() {
+        use spki::DynSignatureAlgorithmIdentifier;
+
+        let priv_key = get_private_key();
+
+        // 255 is the largest representable salt length and must still round-trip.
+        let ok_key = SigningKey::<Sha1>::new_with_salt_len(priv_key.clone(), 255);
+        assert!(ok_key.signature_algorithm_identifier().is_ok());
+
+        // 256 previously truncated to 0; it must now be rejected rather than silently mis-encoded.
+        let bad_key = SigningKey::<Sha1>::new_with_salt_len(priv_key.clone(), 256);
+        assert!(bad_key.signature_algorithm_identifier().is_err());
+
+        // The blinded signing key funnels through the same helper and must behave identically.
+        let bad_blinded = BlindedSigningKey::<Sha1>::new_with_salt_len(priv_key, 256);
+        assert!(bad_blinded.signature_algorithm_identifier().is_err());
+    }
 }

src/pss/blinded_signing_key.rs

@@ -184,7 +184,7 @@ where
     D: Digest + AssociatedOid,
 {
     fn signature_algorithm_identifier(&self) -> spki::Result<AlgorithmIdentifierOwned> {
-        get_pss_signature_algo_id::<D>(self.salt_len as u8)
+        get_pss_signature_algo_id::<D>(self.salt_len)
     }
 }
 

src/pss/signing_key.rs

@@ -221,7 +221,7 @@ where
     D: Digest + AssociatedOid,
 {
     fn signature_algorithm_identifier(&self) -> spki::Result<AlgorithmIdentifierOwned> {
-        get_pss_signature_algo_id::<D>(self.salt_len as u8)
+        get_pss_signature_algo_id::<D>(self.salt_len)
     }
 }