Java RSA BadPaddingException

Question

I'm trying to perform a Diffie-Hellman Key Exchange with a Client and Server, but for some reason when I go to decrypt the message that is being sent to the server. I get the following exception:

Unable to decrypt ciphertext. -> Decryption error
javax.crypto.BadPaddingException: Decryption error
    at java.base/sun.security.rsa.RSAPadding.unpadV15(RSAPadding.java:378)
    at java.base/sun.security.rsa.RSAPadding.unpad(RSAPadding.java:290)
    at java.base/com.sun.crypto.provider.RSACipher.doFinal(RSACipher.java:359)
    at java.base/com.sun.crypto.provider.RSACipher.engineDoFinal(RSACipher.java:392)
    at java.base/javax.crypto.Cipher.doFinal(Cipher.java:2205)
    at server.Server.decrypt(Server.java:354)
    at server.Server.performDHKEX(Server.java:242)
    at server.Server.<init>(Server.java:145)
    at server.Driver.main(Driver.java:15)

I know that my Encryption/Decryption methods work just fine, because if I do something like this: decrypt(encrypt(bytes)), no exception is thrown. Does anyone know why this is happening? Below is my Client and Server code.

TIA!

CLIENT

private SecretKey performDHKEX()
    {   
        SecretKey sharedKey = null;
        
        try
        {
            // Send a message telling the server to get ready for a DH-KEX.
            outputStream.writeObject("DO_DH_KEX");
            
            KeyAgreement ecdhKex = KeyAgreement.getInstance("ECDH");
            KeyPairGenerator gen = KeyPairGenerator.getInstance("EC");
            KeyPair keyPair = gen.genKeyPair();
            
            // Send the Client's Public Value.
            byte[] cText = encrypt(keyPair.getPublic().getEncoded());
            decrypt(encrypt(keyPair.getPublic().getEncoded()));
            byte[] sign = sign(keyPair.getPublic().getEncoded());
            outputStream.writeObject(cText);
            outputStream.writeObject(X509_CERTIFICATE);
            
            
            // Start the exchange with Private Key.
            ecdhKex.init(keyPair.getPrivate());
            
            /* Wait for the response from the Server, then decrypt and load the
            public value the Server sent. */
            byte[][] recvCipherText = (byte[][]) inputStream.readObject();  
            X509Certificate serverCert = (X509Certificate) inputStream
                    .readObject();
            byte[] plainText = decrypt(recvCipherText[0]);
            
            // If the signature verifies, build a keyspec from the public value.
            if (verify(recvCipherText[1], plainText, serverCert))
            {
                // Create a new PublicKey from the Server's Public Value.
                PublicKey pub = KeyFactory.getInstance("EC")
                        .generatePublic(new X509EncodedKeySpec(plainText));
               
                // Perform the last step of the KEX and create a new SecretKey.
                ecdhKex.doPhase(pub, true);
                
                // Use the first 16 bytes of the generate secret for a key.
                sharedKey = new SecretKeySpec(Arrays.copyOfRange(ecdhKex
                        .generateSecret(), 0, 16), "AES");
            }
            else
                System.err.println("Error verifying signature.");
            
        } catch (IOException | ClassNotFoundException | IllegalStateException | 
                InvalidKeyException | NoSuchAlgorithmException | 
                InvalidKeySpecException ex)
        {
            System.err.println("Error during DH-KEX -> " + ex.getMessage());
        }
        
        System.out.println(Base64.getEncoder().encodeToString(sharedKey.getEncoded()));
        return sharedKey;
    }

SERVER

private SecretKey performDHKEX()
    {      
        SecretKey sharedKey = null;
        
        try
        {
            KeyAgreement ecdhKex = KeyAgreement.getInstance("ECDH");
            KeyPairGenerator gen = KeyPairGenerator.getInstance("EC");
            KeyPair keyPair = gen.genKeyPair();
            
            byte[] recvMessage = (byte[]) inputStream.readObject();
            X509Certificate clientCert = (X509Certificate) inputStream
                    .readObject();
            
            // Decrypt the ciphertext.
            byte[] plainText = decrypt(recvMessage); // EXCEPTION THROWN HERE
             
            // If the signature and certificate verify, complete the KEX.
            if (verify(recvMessage, plainText, clientCert))
            {
                System.out.println("inside verify");
                // Send the Server's Public Value.
                byte[] sign = sign(keyPair.getPublic().getEncoded());
                byte[] cText = encrypt(keyPair.getPublic().getEncoded());  
                byte[][] sendArray = {cText, sign};
                outputStream.writeObject(sendArray);
                outputStream.writeObject(X509_CERTIFICATE);
                
                PublicKey pub = KeyFactory.getInstance("EC")
                        .generatePublic(new X509EncodedKeySpec(plainText));
                
                // Start the exchange with the server's Private Value and then
                // doPhase with the Client's public Value.
                ecdhKex.init(keyPair.getPrivate());
                ecdhKex.doPhase(pub, true);
                
                // Use the first 16 bytes of the generate secret for a key.
                sharedKey = new SecretKeySpec(Arrays.copyOfRange(ecdhKex
                        .generateSecret(), 0, 16), "AES");
            }
            
        } catch (IOException | ClassNotFoundException | IllegalStateException | 
                InvalidKeyException | NoSuchAlgorithmException | 
                InvalidKeySpecException ex)
        {
            System.err.println("Error during DH-KEX -> " + ex.getMessage());
        }
        
        System.out.println(Base64.getEncoder().encodeToString(sharedKey.getEncoded()));
        
        return sharedKey;
    }

ENCRYPT

 private byte[] encrypt(byte[] message)
    {
        byte[] cipherText = null;
        
        try
        {
            rsaCipher.init(Cipher.ENCRYPT_MODE, PRIV_KEY);
            cipherText = rsaCipher.doFinal(message);
        } catch (InvalidKeyException | BadPaddingException | 
                IllegalBlockSizeException ex)
        {
            System.err.println("Error encrypting message. -> " + ex
                    .getMessage());
            ex.printStackTrace();
        }
        
        return cipherText;
    }

DECRYPT

private byte[] decrypt(byte[] cipherText)
    {
        try
        {
            rsaCipher.init(Cipher.DECRYPT_MODE, PUB_KEY);
            return rsaCipher.doFinal(cipherText);
        } catch (InvalidKeyException | IllegalBlockSizeException | 
                BadPaddingException ex)
        {
            System.err.println("Unable to decrypt ciphertext. -> " + ex
                    .getMessage());
            ex.printStackTrace();
        }
        
        return null;
    }

For clarification, the RSA Cipher is RSA/ECB/PKCS1Padding and the PRIV_KEY/PUB_KEY is generated from KeyTool

Answer 1

BadPaddingExceptions are a very common failure mode when a decryption is performed with the wrong key.

The server must decode with the client's public key whilst encoding with its own key.

Similarly the client must decode with the server's public key and encode with its own key.

The test you performed suggests that the decode and encode are using the local public key, not the sender's public key.