phpseclib not verifying a signature generated in window.subtlecrypto

Question

So yeah my boss wants to put encryption into his system and he wants messages signed in js and verified in php. currently, I'm using mozilla's subtlecrypto api to generate RSA-PSS keys and sign and phpseclib to verify. The thing is, it doesn't.

Using the js keys, phpseclib can sign and verify just fine, but it can't process the js signature.

Here's my code. JS:

    function keys(){
        var cryptoObj = window.crypto || window.msCrypto;
        let msg = '///';
        if(!cryptoObj)
        {
            alert("Crypto API is not supported by the Browser");
            return;
        }

        window.crypto.subtle.generateKey({
            name: "RSA-PSS",
            modulusLength: 2048, //can be 1024, 2048, or 4096
            publicExponent: new Uint8Array([0x01, 0x00, 0x01]),
            hash: {name: "SHA-256"}, //can be "SHA-1", "SHA-256", "SHA-384", or "SHA-512"
        },
        true, //whether the key is extractable (i.e. can be used in exportKey)
        ["sign", "verify"] //can be any combination of "sign" and "verify"
    )
        .then(function(key) {
            publicKey = key.publicKey;
            privateKey = key.privateKey;
            // For Demo Purpos Only Exported in JWK format
            if (document.getElementById('public').value == "") {
                window.crypto.subtle.exportKey("spki", key.publicKey).then(
                function (keydata) {
                    publicKeyhold = keydata;
                    let exported = publicKeyhold;
                    const exportedAsString = ab2str(exported);
                    const exportedAsBase64 = window.btoa(exportedAsString);
                    const pemExported = `-----BEGIN PUBLIC KEY-----\n${exportedAsBase64}\n-----END PUBLIC KEY-----`;
                    document.getElementById('public').value = pemExported;
                }
            );
            }

            if (document.getElementById('private').value == "" ) {
                msg = document.getElementById('msg').value;
                window.crypto.subtle.exportKey("pkcs8", key.privateKey).then(
                function (keydata) {
                    privateKeyhold = keydata;
                    const priv = privateKeyhold;
                    const privExportedAsString = ab2str(priv);
                    const privExportedAsBase64 = window.btoa(privExportedAsString);
                    const privPemExported = `-----BEGIN RSA PRIVATE KEY-----\n${privExportedAsBase64}\n-----END RSA PRIVATE KEY-----`;
                    document.getElementById('privJSON').data = key.privateKey;
                    document.getElementById('private').value = privPemExported;
                }
            );
        }})
            window.crypto.subtle.sign({
                    name: "RSA-PSS",
                    saltLength: 128, //the length of the salt
                },
                //from generateKey or importKey above
                document.getElementById('privJSON').data,
                getMessageEncoding())
    //ArrayBuffer of data you want to sign
            .then(function(signature) {
                    //returns an ArrayBuffer containing the signature
                    console.dir(ab2str(signature));
                    document.getElementById("cryptmsg").value = window.btoa(ab2str(signature)) ;
                })


}
function ab2str(buf) {
    return String.fromCharCode.apply(null, new Uint8Array(buf));
}


function asciiToUint8Array(str) {
    var chars = [];
    for (var i = 0; i < str.length; ++i)
        chars.push(str.charCodeAt(i));
    return new Uint8Array(chars);
}

function bytesToHexString(bytes) {
    if (!bytes)
        return null;

    bytes = new Uint8Array(bytes);
    var hexBytes = [];

    for (var i = 0; i < bytes.length; ++i) {
        var byteString = bytes[i].toString(16);
        if (byteString.length < 2)
            byteString = "0" + byteString;
        hexBytes.push(byteString);
    }

    return hexBytes.join("");
}
function getMessageEncoding() {
    const messageBox = document.getElementById('msg');
    let message = messageBox.value;
    let enc = new TextEncoder();
    return enc.encode(message);
}

php:

<?php
include('Crypt/RSA.php');
define('CRYPT_RSA_PKCS15_COMPAT', true);

$pubK = $_POST['public'];
$privK = $_POST['private'];
echo $pubK."<br><br>";
echo $privK."<br><br>";

$sign = ($_POST['cryptmsg']);
$txt = $_POST['msg'];

$rsa = new Crypt_RSA();

$rsa->setSignatureMode(CRYPT_RSA_SIGNATURE_PSS);
$rsa->setSaltLength(128);
$rsa->setMGFHash("SHA256");
$rsa->setHash("SHA256");

$rsa->loadKey($privK);
$serverSign = $rsa->sign($txt);

$rsa->loadKey($pubK);
echo "text: ".$txt;
echo "<br><br>js sign: <br>".($sign)."<br><br>";
echo "seclib sign: <br>".base64_encode($serverSign);
echo "<br><br>";

$jsTest = $rsa->verify($txt, base64_decode($sign)) ? 'verified' : 'unverified';
$selfTest = $rsa->verify($txt, $serverSign) ? 'verified' : 'unverified';

echo 'js signature: '. $jsTest;
echo'<br> phpseclib signature (same keys): '. $selfTest;

Answer 1

In the PHP-code, the digest must be specified with sha256, i.e. with lowercase letters instead of uppercase letters:

$rsa->setMGFHash("sha256");
$rsa->setHash("sha256");

phpseclib doesn't know the uppercase name and therefore uses the default digest sha1 instead, [1]. This can easily be tested by outputting the name of the digest used with $rsa->hashName.

The bug isn't that easy to find and maybe it would be better if phpseclib would display an error message or warning than silently using the default.

As a side note: In the JavaScript-code, the private key is exported in the PKCS8-format, but headers and footers of the PKCS1-format are used, [2]. However, phpseclib tolerates this inconsistency.