Windows api AES256 Decryption

Question

I'm not very good with cryptography api's and im trying to encrypt and decrypt a binary file. I put together the following code to encrypt a file and store it back to a file and it works,

#include <Windows.h>
#include <Wincrypt.h>
#include <iostream>
#include <vector>

bool EncryptAES256(char* data, DWORD dataSize, const char* key, DWORD keySize) {
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTHASH hHash = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwMode = CRYPT_MODE_CBC;

    // Create a cryptographic provider context handle.
    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        std::cerr << "Error: CryptAcquireContext failed with error code " << GetLastError() << std::endl;
        return false;
    }

    // Create a hash object of the key.
    if (!CryptCreateHash(hCryptProv, CALG_SHA_256, 0, 0, &hHash)) {
        std::cerr << "Error: CryptCreateHash failed with error code " << GetLastError() << std::endl;
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    if (!CryptHashData(hHash, (const BYTE*)key, keySize, 0)) {
        std::cerr << "Error: CryptHashData failed with error code " << GetLastError() << std::endl;
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Derive a symmetric key from the hash.
    if (!CryptDeriveKey(hCryptProv, CALG_AES_256, hHash, 0, &hKey)) {
        std::cerr << "Error: CryptDeriveKey failed with error code " << GetLastError() << std::endl;
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Set encryption mode to CBC
    if (!CryptSetKeyParam(hKey, KP_MODE, (BYTE*)&dwMode, 0)) {
        std::cerr << "Error: CryptSetKeyParam failed with error code " << GetLastError() << std::endl;
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Apply PKCS7 padding to the data buffer
    int padding = 16 - (dataSize % 16);
    for (int i = 0; i < padding; ++i) {
        data[dataSize + i] = padding;
    }
    dataSize += padding;

    // Encrypt the data using AES 256.
    if (!CryptEncrypt(hKey, 0, TRUE, 0, (BYTE*)data, &dataSize, dataSize * 2)) {
        std::cerr << "Error: CryptEncrypt failed with error code " << GetLastError() << std::endl;
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Print the encrypted data
    std::cout << "Encrypted Data: ";
    for (DWORD i = 0; i < 8; ++i) {
        printf("%02x", (unsigned char)data[i]);
    }
    std::cout << std::endl;

    // Clean up resources.
    CryptDestroyKey(hKey);
    CryptDestroyHash(hHash);
    CryptReleaseContext(hCryptProv, 0);

    return true;
}

int main() {
    HANDLE hF = CreateFileA("C:\\temp\\test.bin",
        GENERIC_READ,
        0,
        NULL,
        OPEN_EXISTING,
        FILE_ATTRIBUTE_NORMAL,
        NULL);
    if (hF == INVALID_HANDLE_VALUE) {
        std::cout << "ERR CreateFileA failed" << std::endl;
        std::cout << GetLastError() << std::endl;
        return -1;
    }

    long r_dwSize = GetFileSize(hF, NULL);
    LPVOID lpbase = VirtualAlloc(NULL, r_dwSize * 2, MEM_COMMIT, PAGE_READWRITE);

    DWORD dwReadBytes = 0;
    BOOL r_read = ReadFile(hF, lpbase, r_dwSize, &dwReadBytes, NULL);
    if (!r_read) {
        std::cout << "ERR ReadFile failed" << std::endl;
        CloseHandle(hF);
        VirtualFree(lpbase, 0, MEM_RELEASE);
        return -1;
    }


    std::cout << "Original Data: ";
    for (DWORD i = 0; i < 8; ++i) {
        printf("%02x", (unsigned char)((char*)lpbase)[i]);
    }
    std::cout << std::endl;

    // Create a new file to write the encrypted data
    HANDLE hEncryptedFile = CreateFileA("C:\\temp\\encrypted_output.bin",
        GENERIC_WRITE,
        0,
        NULL,
        CREATE_ALWAYS, // Creates a new file; if the file exists, it's truncated
        FILE_ATTRIBUTE_NORMAL,
        NULL);

    if (hEncryptedFile == INVALID_HANDLE_VALUE) {
        std::cerr << "ERR CreateFileA for encrypted file failed with error code " << GetLastError() << std::endl;
        CloseHandle(hF);
        VirtualFree(lpbase, 0, MEM_RELEASE);
        return -1;
    }

    // Encrypt the data
    std::string key = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // Replace with your 32-byte key in hexadecimal format
    bool encryptionResult = EncryptAES256((char*)lpbase, dwReadBytes, key.c_str(), key.size() / 2); // Divide by 2 for hexadecimal bytes

    if (encryptionResult) {
        std::cout << "Encryption successful!" << std::endl;
        // Write the encrypted data to the new file
        DWORD dwBytesWritten;
        BOOL writeResult = WriteFile(hEncryptedFile, lpbase, dwReadBytes, &dwBytesWritten, NULL);
        if (writeResult) {
            std::cout << "Encrypted data written to encrypted_output.bin" << std::endl;
        }
        else {
            std::cerr << "Error writing encrypted data to file with error code " << GetLastError() << std::endl;
        }
    }
    else {
        std::cerr << "Encryption failed!" << std::endl;
    }

    // Close the handles and free the allocated memory
    CloseHandle(hEncryptedFile);
    CloseHandle(hF);
    VirtualFree(lpbase, 0, MEM_RELEASE);
    return 0;

}

but when i want to decrypt it i keep getting 2148073477 error code from CryptDecrypt function, below is the code for decryption,

#include <Windows.h>
#include <Wincrypt.h>
#include <iostream>
#include <vector>

bool DecryptAES256(char* data, DWORD& dataSize, const char* key, DWORD keySize) {
    HCRYPTPROV hCryptProv = NULL;
    HCRYPTHASH hHash = NULL;
    HCRYPTKEY hKey = NULL;
    DWORD dwMode = CRYPT_MODE_CBC;

    // Create a cryptographic provider context handle.
    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        std::cerr << "Error: CryptAcquireContext failed with error code " << GetLastError() << std::endl;
        return false;
    }

    // Create a hash object of the key.
    if (!CryptCreateHash(hCryptProv, CALG_SHA_256, 0, 0, &hHash)) {
        std::cerr << "Error: CryptCreateHash failed with error code " << GetLastError() << std::endl;
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    if (!CryptHashData(hHash, (const BYTE*)key, keySize, 0)) {
        std::cerr << "Error: CryptHashData failed with error code " << GetLastError() << std::endl;
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Derive a symmetric key from the hash.
    if (!CryptDeriveKey(hCryptProv, CALG_AES_256, hHash, 0, &hKey)) {
        std::cerr << "Error: CryptDeriveKey failed with error code " << GetLastError() << std::endl;
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Set decryption mode to CBC
    if (!CryptSetKeyParam(hKey, KP_MODE, (BYTE*)&dwMode, 0)) {
        std::cerr << "Error: CryptSetKeyParam failed with error code " << GetLastError() << std::endl;
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Decrypt the data using AES 256.
    if (!CryptDecrypt(hKey, 0, TRUE, 0, (BYTE*)data, &dataSize)) {
        std::cerr << "Error: CryptDecrypt failed with error code " << GetLastError() << std::endl;
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Remove PKCS7 padding
    BYTE padValue = data[dataSize - 1];
    if (padValue >= 1 && padValue <= 16) {
        dataSize -= padValue;
    }
    else {
        std::cerr << "Error: Invalid PKCS7 padding value" << std::endl;
        CryptDestroyKey(hKey);
        CryptDestroyHash(hHash);
        CryptReleaseContext(hCryptProv, 0);
        return false;
    }

    // Clean up resources.
    CryptDestroyKey(hKey);
    CryptDestroyHash(hHash);
    CryptReleaseContext(hCryptProv, 0);

    return true;
}


int main() {
    // Open the encrypted file
    HANDLE hEncryptedFile = CreateFileA("C:\\temp\\encrypted_output.bin",
        GENERIC_READ,
        0,
        NULL,
        OPEN_EXISTING,
        FILE_ATTRIBUTE_NORMAL,
        NULL);

    if (hEncryptedFile == INVALID_HANDLE_VALUE) {
        std::cerr << "Error: Unable to open encrypted file. Error code: " << GetLastError() << std::endl;
        return -1;
    }

    // Get the size of the encrypted file
    DWORD encryptedFileSize = GetFileSize(hEncryptedFile, NULL);

    // Allocate memory to read the encrypted data
    std::vector<BYTE> encryptedData(encryptedFileSize);
    DWORD dwBytesRead;
    if (!ReadFile(hEncryptedFile, encryptedData.data(), encryptedFileSize, &dwBytesRead, NULL)) {
        std::cerr << "Error reading encrypted data from file. Error code: " << GetLastError() << std::endl;
        CloseHandle(hEncryptedFile);
        return -1;
    }

    CloseHandle(hEncryptedFile);

    // Decrypt the data
    std::string key = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
    DWORD dataSize = dwBytesRead; // Initial data size equals the read size

    if (DecryptAES256(reinterpret_cast<char*>(encryptedData.data()), dataSize, key.c_str(), key.size() / 2)) {
        std::cout << "Decryption successful!" << std::endl;

        // Create a new file to write the decrypted data
        HANDLE hDecryptedFile = CreateFileA("C:\\temp\\decrypted_output.exe",
            GENERIC_WRITE,
            0,
            NULL,
            CREATE_ALWAYS,
            FILE_ATTRIBUTE_NORMAL,
            NULL);

        if (hDecryptedFile == INVALID_HANDLE_VALUE) {
            std::cerr << "Error: Unable to create decrypted file. Error code: " << GetLastError() << std::endl;
            return -1;
        }

        // Write the decrypted data to the new file
        DWORD dwBytesWritten;
        if (!WriteFile(hDecryptedFile, encryptedData.data(), dataSize, &dwBytesWritten, NULL)) {
            std::cerr << "Error writing decrypted data to file. Error code: " << GetLastError() << std::endl;
        }
        else {
            std::cout << "Decrypted data written to decrypted_output.exe" << std::endl;
        }

        CloseHandle(hDecryptedFile);
    }
    else {
        std::cerr << "Decryption failed!" << std::endl;
    }

    return 0;
}

unfortunately i cannot figure out how to fix it! I got pretty confused with padding other details i hope i can get some help from you guys.

thanks

Answer 1

You are handling the padding incorrectly. For a block cipher mode like CBC the default padding of wincrypt is PKCS#7. This can be verified with CryptGetKeyParam in conjunction with KP_PADDING (and changed with CryptSetKeyParam). Therefore, your custom padding is to be removed (in encryption and decryption).
The length of the ciphertext (which is needed e.g. for writing the ciphertext to the file) is stored in pdwDataLen after encryption with CryptEncrypt().
If this is considered, decryption of the generated ciphertext works on my machine (assuming encryption and decryption use the same key, which has already been pointed out in the comments).

Security: Note that CBC uses an IV and that for each encryption a random IV should be generated. The IV is not secret. Therefore, it is usually concatenated with the ciphertext so that it is available during decryption.
If you don't generate an IV, a zero IV is used by default (consisting of 16 0x00 values; this can be verified with CryptGetKeyParam() in conjunction with KP_IV), which leads to a reuse of key/IV pairs for multiple encryptions with the same key and is a vulnerability.
For completeness, note that halving the key size with key.size() / 2 is actually not required.