From 0324a000ecea612a15cd5cc22af4f8fc5bb60e04 Mon Sep 17 00:00:00 2001
From: Tony Arcieri <bascule@gmail.com>
Date: Sun, 18 Oct 2026 15:37:13 -0700
Subject: [PATCH] Implement RFC8017 rules for PSS message width

---
 src/algorithms/pss.rs | 12 ++----------
 src/pss.rs            | 23 +++++++++++++++++------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/algorithms/pss.rs b/src/algorithms/pss.rs
index 42dbf33d..a524e527 100644
--- a/src/algorithms/pss.rs
+++ b/src/algorithms/pss.rs
@@ -272,18 +272,14 @@ pub(crate) fn emsa_pss_verify<D>(
     em: &mut [u8],
     s_len: Option<usize>,
     hash: &mut D,
-    key_bits: usize,
+    em_bits: usize,
 ) -> Result<()>
 where
     D: Digest + FixedOutputReset,
 {
-    let em_bits = key_bits - 1;
     let em_len = em_bits.div_ceil(8);
-    let key_len = key_bits.div_ceil(8);
     let h_len = <D as Digest>::output_size();
 
-    let em = &mut em[key_len - em_len..];
-
     let (db, h) = emsa_pss_verify_pre(m_hash, em, em_bits, s_len, h_len)?;
 
     // 7. Let dbMask = MGF(H, em_len - h_len - 1)
@@ -328,18 +324,14 @@ pub(crate) fn emsa_pss_verify_digest<D>(
     m_hash: &[u8],
     em: &mut [u8],
     s_len: Option<usize>,
-    key_bits: usize,
+    em_bits: usize,
 ) -> Result<()>
 where
     D: Digest + FixedOutputReset,
 {
-    let em_bits = key_bits - 1;
     let em_len = em_bits.div_ceil(8);
-    let key_len = key_bits.div_ceil(8);
     let h_len = <D as Digest>::output_size();
 
-    let em = &mut em[key_len - em_len..];
-
     let (db, h) = emsa_pss_verify_pre(m_hash, em, em_bits, s_len, h_len)?;
 
     let mut hash = D::new();
diff --git a/src/pss.rs b/src/pss.rs
index 22dea822..52c5b1c9 100644
--- a/src/pss.rs
+++ b/src/pss.rs
@@ -145,7 +145,8 @@ where
     D: Digest + FixedOutputReset,
 {
     let mut em = decode_signature(pub_key, sig)?;
-    emsa_pss_verify(hashed, &mut em, salt_len, digest, pub_key.n().bits() as _)
+    let em_bits = pub_key.n().bits() as usize - 1;
+    emsa_pss_verify(hashed, &mut em, salt_len, digest, em_bits)
 }
 
 pub(crate) fn verify_digest<D>(
@@ -158,24 +159,34 @@ where
     D: Digest + FixedOutputReset,
 {
     let mut em = decode_signature(pub_key, sig)?;
-    emsa_pss_verify_digest::<D>(hashed, &mut em, salt_len, pub_key.n().bits() as _)
+    let em_bits = pub_key.n().bits() as usize - 1;
+    emsa_pss_verify_digest::<D>(hashed, &mut em, salt_len, em_bits)
 }
 
-/// Checks signature length and range before applying the public-key operation.
+/// Performs the length, RSAVP1, and I2OSP steps of RFC8017 §8.1.2.
 fn decode_signature(pub_key: &RsaPublicKey, signature: &[u8]) -> Result<Vec<u8>> {
-    // Step 1: S must be exactly k octets, independently of integer storage precision.
+    // Step 1: `S` must be exactly `k` octets, independently of integer storage precision.
     if signature.len() != pub_key.size() {
         return Err(Error::Verification);
     }
 
-    // Steps 2a and 2b: s = OS2IP(S), m = RSAVP1((n, e), s).
+    // Steps 2a and 2b: `s = OS2IP(S), m = RSAVP1((n, e), s)`.
     let s = os2ip(signature, pub_key.n_bits_precision()).map_err(|_| Error::Verification)?;
     if s >= *pub_key.n().as_ref() {
         return Err(Error::Verification);
     }
     let m = rsa_encrypt(pub_key, &s)?;
 
-    Ok(i2osp(m, pub_key.size()))
+    // Step 2c: EM has `emLen` octets, which can be `k - 1`.
+    //
+    // Check the public representative's width before calling infallible I2OSP; discarding a nonzero
+    // leading octet is invalid.
+    let em_bits = pub_key.n().bits() as usize - 1;
+    let em_len = em_bits.div_ceil(8);
+    if m.bits() as usize > 8 * em_len {
+        return Err(Error::Verification);
+    }
+    Ok(i2osp(m, em_len))
 }
 
 /// SignPSS calculates the signature of hashed using RSASSA-PSS.