Fix RC2 effective-key-length mask per RFC 2268

CHANGELOG.md

@@ -1,6 +1,14 @@
 Forge ChangeLog
 ===============
 
+## 1.4.1 - 2026-xx-xx
+
+### Fixed
+- [rc2] Compute the effective-key-length mask as RFC 2268 defines it. The mask
+  was shifting out `T1 MOD 8` bits instead of `8*T8 - T1`, so key expansion was
+  wrong whenever the effective key size was not a multiple of 8 bits. Two of
+  the eight RFC 2268 section 5 test vectors failed; both now pass.
+
 ## 1.4.0 - 2026-03-24
 
 ### Security

lib/rc2.js

@@ -81,7 +81,10 @@ forge.rc2.expandKey = function(key, effKeyBits) {
   var T = key.length();
   var T1 = effKeyBits;
   var T8 = Math.ceil(T1 / 8);
-  var TM = 0xff >> (T1 & 0x07);
+  /* RFC 2268: TM = 255 MOD 2^(8 + T1 - 8*T8), ie. TM has its
+     8 - (8*T8 - T1) least significant bits set. The number of bits shifted
+     out is therefore 8*T8 - T1, which is (-T1) MOD 8, not T1 MOD 8. */
+  var TM = 0xff >> (-T1 & 0x07);
   var i;
 
   for(i = T; i < 128; i++) {

tests/unit/jsbn.js

@@ -1,7 +1,7 @@
 var ASSERT = require('assert');
 var JSBN = require('../../lib/jsbn');
 
-describe.only('jsbn', function() {
+describe('jsbn', function() {
   describe('GHSA-5m6q-g25r-mvwx', function() {
     // regression tests for GHSA-5m6q-g25r-mvwx
     // test BigInteger.modInverse does not infinite loop with 0 inputs.

tests/unit/rc2.js

@@ -84,5 +84,41 @@ var UTIL = require('../../lib/util');
       cipher.finish();
       ASSERT.equal(cipher.output, 'revolution');
     });
+
+    // RFC 2268 section 5 test vectors. Only the first output block is
+    // compared, since forge appends a PKCS#7 padding block.
+    function rc2Vector(keyHex, effKeyBits, plainHex, expectedHex) {
+      var cipher = RC2.createEncryptionCipher(
+        UTIL.hexToBytes(keyHex), effKeyBits);
+      cipher.start(null);
+      cipher.update(new UTIL.createBuffer(UTIL.hexToBytes(plainHex)));
+      cipher.finish();
+      ASSERT.equal(cipher.output.toHex().substr(0, 16), expectedHex);
+    }
+
+    it('should match RFC 2268 vector w/8 byte key, 63 effective bits', function() {
+      rc2Vector('0000000000000000', 63, '0000000000000000',
+        'ebb773f993278eff');
+    });
+
+    it('should match RFC 2268 vector w/8 byte key, 64 effective bits', function() {
+      rc2Vector('ffffffffffffffff', 64, 'ffffffffffffffff',
+        '278b27e42e2f0d49');
+    });
+
+    it('should match RFC 2268 vector w/1 byte key, 64 effective bits', function() {
+      rc2Vector('88', 64, '0000000000000000', '61a8a244adacccf0');
+    });
+
+    it('should match RFC 2268 vector w/16 byte key, 128 effective bits', function() {
+      rc2Vector('88bca90e90875a7f0f79c384627bafb2', 128,
+        '0000000000000000', '2269552ab0f85ca6');
+    });
+
+    it('should match RFC 2268 vector w/33 byte key, 129 effective bits', function() {
+      rc2Vector(
+        '88bca90e90875a7f0f79c384627bafb216f80a6f85920584c42fceb0be255daf1e',
+        129, '0000000000000000', '5b78d3a43dfff1f1');
+    });
   });
 })();