C++ AES OpenSSL

Question

I'm trying to read a string from a file encrypt it with AES and then save it to other file. Later I need to read the new file, decrypt and save the output to a new file again. The problem is some strange character are appearing.

int Crypt::__aesEncrypt(const unsigned char *msg, size_t msgLen, unsigned char **encMsg) {
EVP_CIPHER_CTX *aesEncryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX));
EVP_CIPHER_CTX_init(aesEncryptCtx);

unsigned char *aesKey = (unsigned char*)malloc(AES_KEYLEN/8);
unsigned char *aesIV = (unsigned char*)malloc(AES_KEYLEN/8);

unsigned char *aesPass = (unsigned char*)malloc(AES_KEYLEN/8);
unsigned char *aesSalt = (unsigned char*)malloc(8);

if(RAND_bytes(aesPass, AES_KEYLEN/8) == 0) {
    return FAILURE;
}

if(RAND_bytes(aesSalt, 8) == 0) {
    return FAILURE;
}

if(EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha1(), aesSalt, aesPass, AES_KEYLEN/8, AES_ROUNDS, aesKey, aesIV) == 0) {
    return FAILURE;
}

strncpy((char*)aesKey, (const char*)"B374A26A714904AAB374A26A714904AA", AES_KEYLEN/8);
strncpy((char*)aesIV, (const char*)"B374A26A714904AA", AES_KEYLEN/16);

size_t blockLen = 0;
size_t encMsgLen = 0;

*encMsg = (unsigned char*)malloc(msgLen + AES_BLOCK_SIZE);
if(encMsg == NULL) return FAILURE;
(*encMsg)[0] = '\0';

if(!EVP_EncryptInit_ex(aesEncryptCtx, EVP_aes_256_cbc(), NULL, aesKey, aesIV)) {
    return FAILURE;
}

if(!EVP_EncryptUpdate(aesEncryptCtx, *encMsg, (int*)&blockLen, (unsigned char*)msg, msgLen)) {
    return FAILURE;
}
encMsgLen += blockLen;

if(!EVP_EncryptFinal_ex(aesEncryptCtx, *encMsg + encMsgLen, (int*)&blockLen)) {
    return FAILURE;
}

EVP_CIPHER_CTX_cleanup(aesEncryptCtx);
free(aesEncryptCtx);

free(aesKey);
free(aesIV);

return encMsgLen + blockLen;
}

int Crypt::__aesDecrypt(unsigned char *encMsg, size_t encMsgLen, char **decMsg) {
EVP_CIPHER_CTX *aesDecryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX));
EVP_CIPHER_CTX_init(aesDecryptCtx);

unsigned char *aesKey;
unsigned char *aesIV;

aesKey = (unsigned char*)malloc(AES_KEYLEN/8);
aesIV = (unsigned char*)malloc(AES_KEYLEN/8);

unsigned char *aesPass = (unsigned char*)malloc(AES_KEYLEN/8);
unsigned char *aesSalt = (unsigned char*)malloc(8);

if(RAND_bytes(aesPass, AES_KEYLEN/8) == 0) {
    return FAILURE;
}

if(RAND_bytes(aesSalt, 8) == 0) {
    return FAILURE;
}

if(EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha1(), aesSalt, aesPass, AES_KEYLEN/8, AES_ROUNDS, aesKey, aesIV) == 0) {
    return FAILURE;
}

strncpy((char*)aesKey, (const char*)"B374A26A714904AAB374A26A714904AA", AES_KEYLEN/8);
strncpy((char*)aesIV, (const char*)"B374A26A714904AA", AES_KEYLEN/16);

size_t decLen = 0;
size_t blockLen = 0;

*decMsg = (char*)malloc(encMsgLen);
if(*decMsg == NULL) return FAILURE;

if(!EVP_DecryptInit_ex(aesDecryptCtx, EVP_aes_256_cbc(), NULL, aesKey, aesIV)) {
    return FAILURE;
}

if(!EVP_DecryptUpdate(aesDecryptCtx, (unsigned char*)*decMsg, (int*)&blockLen, encMsg, (int)encMsgLen)) {
    return FAILURE;
}
decLen += blockLen;

if(!EVP_DecryptFinal_ex(aesDecryptCtx, (unsigned char*)*decMsg + decLen, (int*)&blockLen)) {
    return FAILURE;
}
decLen += blockLen;

(*decMsg)[decLen] = '\0';

EVP_CIPHER_CTX_cleanup(aesDecryptCtx);

return encMsgLen;
}

Encrypting:

    unsigned char *encMsg = NULL;
    int size = __aesEncrypt((const unsigned char*)decrypted_string.c_str(), decrypted_string.size(), &encMsg);


    return std::string(reinterpret_cast<const char*>(encMsg), size);

Decrypting:

    char *decMsg = NULL;
    int size = __aesDecrypt((unsigned char*)encrypted_string.c_str(), encrypted_string.size(), &decMsg);

    return std::string(reinterpret_cast<const char*>(decMsg), size);

I can successfully crypt and decrypt, but some strange characters are appearing at the end of the encrypted file, they are like empty spaces:

Answer 1

AES is a block cypher. It takes blocks of 16 bytes, and encrypt them into a block of 16 byets. If you try to use it with data whose length is not a multiple of 16, padding (usually random data) is added to round it up to a multiple of 16 bytes. You need to manage the length of the data yourself.

Example:

int encryptHelper(const string& msg, ...)
{
    uint32_t msgSize = msg.length();
    newMsg.push_back((msgSize >> 0)  & 0xFF);
    newMsg.push_back((msgSize >> 8)  & 0xFF);
    newMsg.push_back((msgSize >> 16) & 0xFF);
    newMsg.push_back((msgSize >> 24) & 0xFF);
    string newMsg(reinterpret_cast<const char*>(&msgSize), sizeof(msgSize));
    newMsg += msg;
    return __aesEncrypt(newMsg.c_str(), newMsg.length(), ...);
}

int decryptHelper(const string& encrypted, ...)
{
    string msg = ... whatever you are doing to decrypt
    uint32_t actualSize = 0;
    // remove signal first, then widen
    actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[0])) << 0;
    actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[1])) << 8;
    actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[2])) << 16;
    actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[3])) << 24;
    string actualMsg = msg.substr(4, actualSize);
    ...
}

I didn't bother to write the exact code to call your functions because all that casting and memory leaking gave me nausea. Fill in the blanks.