C: Can not decrypt message Openssl

Question

I'm new with cryptography, so I decided to create simple program that would open a file encrypt data, put it in etest.txt, then open this file decrypt it and put it indetest.txt. I know it sounds really weired but its for educational purposes. so here is my code.

#include <openssl/rsa.h>
#include <openssl/pem.h>
#include <openssl/err.h>
#include <stdio.h>
#include <string.h>

int main(void) {
size_t pri_len;            // Length of private key
size_t pub_len;            // Length of public key
char   *pri_key;           // Private key
char   *pub_key;           // Public key
char   *msg = malloc(256);  // Message to encrypt
char   *encrypt = NULL;    // Encrypted message
char   *decrypt = NULL;    // Decrypted message
char   *err;               // Buffer for any error messages

// Generate key pair
RSA *keypair = RSA_generate_key(2048, 3, NULL, NULL);
 FILE *in   = fopen("test.txt", "rb");
 FILE *out  = fopen("etest.txt", "wb");

 if(in == NULL)
 {
    printf("in Error is %d (%s).\n", errno, strerror(errno));
 }
 if(out == NULL)
 {
    printf("out Error is %d (%s).\n", errno, strerror(errno));
 }

encrypt = malloc(RSA_size(keypair));
for(;;)
{
    //213 because of padding
    memset(msg, '\0', 256);
    memset(encrypt, '\0', 256);
    fread(msg, 213, 1, in);

    if((RSA_public_encrypt(strlen(msg), (unsigned char*)msg, (unsigned char*)encrypt,
                                         keypair, RSA_PKCS1_OAEP_PADDING)) == -1) {
        ERR_load_crypto_strings();
        ERR_error_string(ERR_get_error(), err);
        fprintf(stderr, "Error encrypting message: %s\n", err);           
    }

     if(fwrite(encrypt, 256, 1, out) != 1)
     {
       printf("fwrite Error is %d (%s).\n", errno, strerror(errno));
     }

    if(feof(in))
    {
        break;
    }      
}

fclose(in);
fclose(out);

in   = fopen("etest.txt", "rb");
out  = fopen("dtest.txt", "wb");

 if(in == NULL)
 {
    printf("in Error is %d (%s).\n", errno, strerror(errno));
 }
 if(out == NULL)
 {
    printf("out Error is %d (%s).\n", errno, strerror(errno));
 }

decrypt = malloc(RSA_size(keypair));

for(;;)
{
     //I use malloc because if i didnt it would from file and if it filled the msg and if this function would execute second time it would not overwrite the whole buffer and would cause problem
     memset(decrypt, '\0', 256);
     memset(msg, '\0', 256);

     fread(msg, 256, 1, in);


     if(RSA_private_decrypt(256, (unsigned char*)msg, (unsigned char*)decrypt,
                           keypair, RSA_PKCS1_OAEP_PADDING) == -1) {
        ERR_load_crypto_strings();
        ERR_error_string(ERR_get_error(), err);
        fprintf(stderr, "Error decrypting message: %s\n", err);
    }

    fwrite(decrypt, 256, 1, out);

    if(feof(in))
    {
      break;
    }
}

fclose(in);
fclose(out);
RSA_free(keypair);
return 0;

}

When I run code it gives me back error saying:Error decrypting message: error:0407A079:rsa routines:RSA_padding_check_PKCS1_OAEP:oaep decoding error but if i delete this codememset(msg, '\0', 256); it shows that everything works fine but it causes problems because msg buffer is overwritten with first few bytes that second fread() function overwrote. Sorry if my questions sound silly. Hope you can help. thanks.

Answer 1

Your are using fwrite(decrypt, 256, 1, out); which is wrong.size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream) Second parameter is the size in bytes of each element to be read And the third one is number of elements, each one with a size of size bytes.