RSA_public_encrypt returning -1 and not able to decrypt, RSA 2048-bit with padding is used for encrypting and decrypting files

Question

In the code, I have a file, which is divided into 256 bytes block in order to encrypt with padding, but when I am calling RSA_public_encrypt, it's returning -1, can't understand what mistake I have done.Can anyone guide me where exactly I am going wrong?

#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>
#include <openssl/bio.h>
#include <openssl/err.h>
#include <stdio.h>

int padding = RSA_PKCS1_PADDING;


RSA * createRSAWithFilename(char * filename,int public)
{
    RSA *rsa= NULL;
    FILE * fp = fopen(filename,"rb");

    if(fp == NULL) {
        printf("Unable to open file %s \n",filename);
        return NULL;
    }
    if(public) {
        rsa =PEM_read_RSA_PUBKEY(fp, &rsa,NULL, NULL);
    }
    else {
        rsa = PEM_read_RSAPrivateKey(fp, &rsa,NULL, NULL);
    }
    if(rsa == NULL) {
        printf( "Failed to create RSA");
    }

    return rsa;
}


int main() {
    FILE * fp = fopen("hello.c", "rb");
    fseek(fp, 0, SEEK_END);
    int file_size = ftell(fp);
    fseek(fp, 0, SEEK_SET);

    char msg_blocks[2048/8];
    unsigned char file_encrypt_buf[4098]= {};
    size_t bytesRead;
    int encrypted_length=0;
    int encrypt_size;
    FILE *append = fopen("out.bin", "a+");
    RSA * rsa1= createRSAWithFilename("public.pem",1);
    if (fp != NULL)
    {
        while ((bytesRead = fread(msg_blocks, sizeof(unsigned char),sizeof(msg_blocks), fp)) > 0)
        {
            printf("bytesread %d\n",bytesRead);
            printf("%d\n",strlen(msg_blocks));
            encrypted_length= RSA_public_encrypt(strlen(msg_blocks),(unsigned char*)msg_blocks,(unsigned char*)file_encrypt_buf,rsa1,padding);
            fwrite(file_encrypt_buf,sizeof(unsigned char),sizeof(file_encrypt_buf),append);
            printf("encrypted length: %d\n",encrypted_length);
        }
        fclose(append);
        fclose(fp);
    }
    printf("Encrypted message written to file.\n");

  //decryption
    append = fopen("out.bin", "r");
    fseek(append, 0, SEEK_END);
    encrypt_size = ftell(append);
    fseek(append, 0, SEEK_SET);
    char decrypt_msg_blocks[2048/8];
    size_t decryptbytesRead;
    int decrypted_length=0;
    unsigned char file_decrypt_buf[4098]={};
    FILE * fp1 = fopen("hellodec.c", "a+");
     RSA * rsa2= createRSAWithFilename("private.pem",0);
    if(append!=NULL) {
        while ((decryptbytesRead = fread(decrypt_msg_blocks, sizeof(unsigned char),sizeof(decrypt_msg_blocks), append)) > 0) {
            decrypted_length=RSA_private_decrypt(strlen(decrypt_msg_blocks),(unsigned char*)decrypt_msg_blocks, (unsigned char*)file_decrypt_buf, rsa2,padding);
            fwrite(file_decrypt_buf,,sizeof(unsigned char), sizeof(file_decrypt_buf), fp1);
        }
        fclose(fp1);
        fclose(append);
    }
    printf("Decrypted message written to file.\n");
    return 0;
}

Getting following error:

bytesread 256
256
encrypted length: -1
bytesread 256
256
encrypted length: -1
bytesread 202
256
encrypted length: -1
encrypted length: -1
Encrypted message written to file.

Answer 1

You can't encrypt a full 256 byte block with 2048 bit key as the padding uses some of the space. A PKCS1 padding uses around 11 bytes. In practice this means that you encrypted data is going to be growing in size.

Generally RSA is not suited for bulk encryption as it's quite slow (more than a factor 1000 when compared to AES). Instead use a symmetric encryption algorithm like AES if you can. If you really need the two key's of RSA, use a Hybrid approach where you encrypt the data with a random symmetric key, and then encrypt that key with the RSA key.

Another benefit of symmetric encryption is that libraries automatically supports bulk encryption, where you don't need to handle chopping your data up into small blocks before encryption.