Why does this code not produce a correct hash?

Question

I had to write a tiny decryption program in C to brute force a key from one file, here "resource.bin", and use that one to decrypt another file using DES-EDE, here "rom_dump.bin". The indication for the correct key is that the decrypted file content starts with a ten digit number terminated by \0. After that the decrypted content should be written to another file, here "decrypted.bin" and the file should be hashed with ECDSA (with the function EVP_ecdsa()). All of this is done on SUSE Linux. The files can be found here:

https://spideroak.com/browse/share/see/stack/StackOverflow/

Now, the decryption works just fine, but the hash value is not correct:

a493af52c1a000fcace34de8b0a74a9cf9067ffc

But even after days of searching, I just cannot find the problem. It's probably just something rediculess I'm overseeing but I'd be very glad if somebody could help me out here. Thanks in advance.

#include <stdio.h>
#include <string.h>
#include <openssl/evp.h>
#include <unistd.h>
#include <fcntl.h>

const unsigned long long bufferSize = 0x10000;

int checkOutput(unsigned char *output) {
    int i = 0;
    for (i; i < 6; i++) {
        if (!isdigit(output[i])) {
            return 0;
        }
    }

    return 1;
}

void changeKey(unsigned char *key, unsigned char *fileContent, long keyLength,
long initVectorLength) {
    int i = 0;
    for (i; i < keyLength + initVectorLength; i++) {
        key[i] = fileContent[i];
    }
}

void toHashFile(FILE *hashFile, unsigned char *hash, int hashLength) {
    int i = 0;
    for (i; i < hashLength; i++) {
        fprintf(hashFile, "%02x", hash[i]);
    }

    fprintf(hashFile, "\n");
}

void toOutputFile(FILE *fileName, unsigned char *output,
int outputLength) {
    int i = 0;
    for (i; i < outputLength; i++) {
        fprintf(fileName, "%c", output[i]);
    }

    fprintf(fileName, "\n");
}

void writeToFile(const unsigned char *fileName, unsigned char *content,
int contentLength,
void (*functionPointer)(FILE *, unsigned char *, int)) {
    FILE *file = fopen(fileName, "w");
    (*functionPointer)(file, content, contentLength);
    fclose(file);
}

void createHash(unsigned char *hash, unsigned char *output, int length,
int *hashLength) {
    EVP_MD_CTX hashContext;
    EVP_MD_CTX_init(&hashContext);
    EVP_DigestInit(&hashContext, EVP_ecdsa());
    EVP_DigestUpdate(&hashContext, output, length);
    EVP_DigestFinal(&hashContext, hash, hashLength);
}

int main() {
    /* output stuff */
    unsigned char keyAndInitVector[24] = {0x00};
    unsigned char output[bufferSize];
    unsigned char outputFinal[bufferSize];
    int outputLength;

    /* determine key length and init vector */
    int initVectorLength = EVP_CIPHER_iv_length(EVP_des_ede_ecb());
    int keyLength = EVP_CIPHER_key_length(EVP_des_ede_ecb());

    /* read resource files */
    unsigned char romFileContent[bufferSize];
    unsigned char resFileContent[bufferSize];
    int romLength = read(open("rom_dump.bin", O_RDONLY), romFileContent,
        bufferSize);
    int resLength = read(open("resource.bin", O_RDONLY), resFileContent,
        bufferSize);

    /* init context */
    EVP_CIPHER_CTX cypherContext;
    EVP_CIPHER_CTX_init(&cypherContext);

    int i = 0, j;
    int isDecrypted = 0;

    for (i; i < romLength - (keyLength + initVectorLength) &&
    !isDecrypted; i++) {
        changeKey(keyAndInitVector, romFileContent + i, keyLength,
            initVectorLength);

        EVP_DecryptInit(&cypherContext, EVP_des_ede_ecb(),
            keyAndInitVector, keyAndInitVector + keyLength);
        EVP_DecryptUpdate(&cypherContext, output, &outputLength,
            resFileContent, resLength);

        for (j = 0; j < resLength; j++) {
            if (checkOutput(output + j) == 1) {
                isDecrypted = 1;
                break;
            }
        }
    }

    if (isDecrypted) {
        int postfixLength;
        EVP_DecryptFinal(&cypherContext, outputFinal,
            &postfixLength);

        writeToFile("decrypted.bin", output,
            outputLength + postfixLength, &toOutputFile);

        int hashLength = 0;
        unsigned char hash[bufferSize];
        createHash(hash, output, outputLength + postfixLength,
            &hashLength);
        writeToFile("hash.txt", hash, hashLength, &toHashFile);
    }

    EVP_CIPHER_CTX_cleanup(&cypherContext);
    return isDecrypted;
}

Answer 1

In your toOutputFile() function, you add a \n to your file, but down in the main() you don't hash the file, but the output.

That means, your decrypted.bin has an additional \n which does not exist in your output which is why when hashing the file, the hash will be different from the one you created with that program.