aes-256-cbc encryption/decryption keys don't work

Question

I made a simple file encryptor/decryptor. It takes the mode and the file to operate with as arguments. When encrypting, it generates random string and encrypts the file using that. When decrypting a file, it prompts the user for password and uses that when decrypting.

My problem is that I get gibberish instead of the plaintext when decrypting even though I carefully write the same key to the input.

I tracked the problem to the passwords since everything worked flawlessly when I used constant password like

unsigned char constkey[] = "asd123";

and used it for both encryption and decryption.

What is making my keys different?

Big thanks for James K Polk for the encryption/decryption code!

#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <openssl/evp.h>
#include <openssl/aes.h>
#include <time.h>
#include <errno.h>

#ifndef TRUE
#define TRUE 1
#endif

#ifndef FALSE
#define FALSE 0
#endif


/* Encrypt or decrypt, depending on flag 'should_encrypt' */
void en_de_crypt(int should_encrypt, FILE *ifp, FILE *ofp, unsigned char *ckey, unsigned char *ivec) {

    const unsigned BUFSIZE=4096;
    unsigned char *read_buf = malloc(BUFSIZE);
    unsigned char *cipher_buf;
    unsigned blocksize;
    int out_len;
    EVP_CIPHER_CTX ctx;

    EVP_CipherInit(&ctx, EVP_aes_256_cbc(), ckey, ivec, should_encrypt);
    blocksize = EVP_CIPHER_CTX_block_size(&ctx);
    cipher_buf = malloc(BUFSIZE + blocksize);

    while (1) {

        // Read in data in blocks until EOF. Update the ciphering with each read.

        int numRead = fread(read_buf, sizeof(unsigned char), BUFSIZE, ifp);
        EVP_CipherUpdate(&ctx, cipher_buf, &out_len, read_buf, numRead);
        fwrite(cipher_buf, sizeof(unsigned char), out_len, ofp);
        if (numRead < BUFSIZE) { // EOF
            break;
        }
    }

    // Now cipher the final block and write it out.

    EVP_CipherFinal(&ctx, cipher_buf, &out_len);
    fwrite(cipher_buf, sizeof(unsigned char), out_len, ofp);

    // Free memory

    free(cipher_buf);
    free(read_buf);
}

int rand_string(char *str, size_t size)
{
    size_t n;
    const char charset[] = "ABCDEFGHIJKLMNOPQRSTUWVXYZO1234567890";

    for (n = 0; n < size; n++) {
        int key = rand() % (int) (sizeof charset - 1);
        str[n] = charset[key];
    }

    str[size] = '\0';
    return 0;
}

void copyfile(FILE* from, FILE* to)
{
    fseek(from, 0, SEEK_END);
    long filesize = ftell(from);
    fseek(from, 0, SEEK_SET);
    char b[2];
    long a;

    for(a=0; a<filesize; a++)
    {
        fread(b, 1, 1, from);
        fwrite(b, 1, 1, to);
    }
}

int main(int argc, char *argv[]) {

    if (argc != 3) { printf("usage: %s mode file\n", argv[0]); return 0; }

    unsigned char ivec[] = "dontusethisinput";
    FILE *fIN, *fOUT;

    if (!strcmp(argv[1],"e"))
    {
        srand(time(NULL)); /* seed random */

        char passkey[6];
        rand_string(passkey,5); /* generate random password */

        if((fIN = fopen(argv[2], "rb")) == NULL){ printf("ERROR: %s\n", strerror(errno)); return 0; }
        fOUT = fopen("tempfile", "wb+");

        en_de_crypt(TRUE, fIN, fOUT, (unsigned char*)passkey, ivec); /* encrypt the file */

        fclose(fIN);
        fclose(fOUT);

        if((fIN = fopen(argv[2], "wb+")) == NULL){ printf("ERROR: %s\n", strerror(errno)); return 0; }
        fOUT = fopen("tempfile", "rb");

        copyfile(fOUT, fIN); /* replace the file with encrypted one */

        fclose(fIN);
        fclose(fOUT);

        printf("file encrypted with key [%s]\n", passkey);
    }
    else if(!strcmp(argv[1],"d"))
    {
        char key[6];
        printf("Password: ");
        fgets(key, 6, stdin); /* prompt for the key */
        key[5] = '\0';

        if((fIN = fopen(argv[2], "rb")) == NULL){ printf("ERROR: %s\n", strerror(errno)); return 0; }
        fOUT = fopen("tempfile", "wb+");

        en_de_crypt(FALSE, fIN, fOUT, (unsigned char*)key, ivec); /* decrypt the file */

        fclose(fIN);
        fclose(fOUT);

        if((fIN = fopen(argv[2], "wb+")) == NULL){ printf("ERROR: %s\n", strerror(errno)); return 0; }
        fOUT = fopen("tempfile", "rb");

        copyfile(fOUT, fIN); /* replace the file with decrypted one */

        fclose(fIN);
        fclose(fOUT);

        printf("file decrypted with key [%s]\n",key);
    }
    else { printf("invalid mode\n"); return 0; }

    remove("tempfile");
    return 0;
}

Answer 1

I tracked the problem to the passwords since everything worked flawlessly when I used constant password like

unsigned char constkey[] = "asd123";

and used it for both encryption and decryption.

AES don't use passwords it uses (binary) keys. In the case of AES-256, it expects a 256 bit long key. As OpenSSL will treat your constkey as a 256 bit key, it will read beyond what you provided and end up with some random data in the end of the key (if you are lucky to avoid any access violations).

If you do this within the same program, you properly end up with the same random data in the key on both encrypt and decrypt - and everything seems fine.

Try with correct key length.