Decrypt function doesn't work as intended

Question

I'm taking a course in fundamental encryption/decryption algorithms, and I'm stuck on this one lab assignment.

We're given a C file called simplecrypt.c that, when run, takes an input file and encrypts it to an output file (the input remains intact). Our assignment is to write a decrypt function in simplecrypt.c as well, so that you can encrypt and decrypt your files, not just encrypt them. And this is where I'm stuck. I don't know if I'm making it too simple or if I took water over my head with this course, but if anyone can shed some light in what I'm doing wrong, I'd greatly appreciate it.

This is how the encrypt function looks like. I believe it's AES, if I'm not mistaken? Please note that the encryption key has been given to us in plain text:

int crypt(unsigned char *key,
             unsigned char *pInputBuf,
             unsigned char *pOutputBuf,
             long nLength)
{
    int nKeyPos = 0;
    long n;
    unsigned char KeyA = 0;

    if((pInputBuf != NULL) && (pOutputBuf != NULL))
    {
        for(n = 0; n < KEY_BUF; n++)
            KeyA ^= key[n];

        nKeyPos = KeyA%KEY_BUF;

        for(n = 0; n < nLength; n++)
        {
            pOutputBuf[n] = pInputBuf[n]^(key[nKeyPos]*KeyA);
            KeyA += pOutputBuf[n];
            nKeyPos = pOutputBuf[n]%KEY_BUF;
        }
        return 0;   // success
    }
    return 1;
}

And this is the decrypt that I made:

int decrypt(unsigned char *key,
             unsigned char *pInputBuf,
             unsigned char *pOutputBuf,
             long nLength)
{
    int nKeyPos = 0;
    long n;
    unsigned char KeyA = 0;

    if((pInputBuf != NULL) && (pOutputBuf != NULL))
    {
        for(n = 0; n < KEY_BUF; n++)
            KeyA ^= key[n];

        nKeyPos = KeyA%KEY_BUF;

        for(n = 0; n < nLength; n++)
        {
            // This is where I made changes.
            pInputBuf[n] = pOutputBuf[n]^(key[nKeyPos]*KeyA);
            KeyA += pInputBuf[n];
            nKeyPos = pInputBuf[n]%KEY_BUF;
        }
        return 0;   // success
    }
    return 1;
}

Basically, all I did was to reverse the pInputBuf with pOutputBuf and vice versa. Why doesn't this work in practice when it works in theory? As I said, any help that could send me down the right direction would be awesome.

This is the full code:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define KEY_BUF 20
#define FNAME_BUF 256


void stripnl(char *str) 
{
    while(strlen(str) && ((str[strlen(str) - 1] == 13) || (str[strlen(str) - 1] == 10))) 
    {
        str[strlen(str) - 1] = 0;
    }
}


long writefile(unsigned char *pOutputBuf, long size) 
{
    FILE *outfile;
    char fname[FNAME_BUF];
    unsigned char *p = pOutputBuf;
    long i=0;

    // Read in the filename
    printf("Enter the name of the output file: ");
    fgets(fname, sizeof(fname), stdin);

    // We need to get rid of the newline char
    stripnl(fname);

    // Open the file. If NULL is returned there was an error, corrected to VS2012 standard
    // if((outfile = fopen(fname, "wb")) == NULL) 
    if((fopen_s(&outfile, fname, "wb")) != 0) 
    {
        printf("Error Opening File.\n");
        return(0);
    }

    // Put characters in outfile
    while( (i < size) && (fputc(*(p++), outfile) != EOF) )
        i++;

    fclose(outfile);  // Close the file
    return i;
}


char* readfile(unsigned char *pInutBuf, long *nLength) 
{
    FILE *infile;
    char fname[FNAME_BUF];
    int ch=0;
    long i;

    // Read in the filename
    printf("Enter the name of the input file: ");
    fgets(fname, sizeof(fname), stdin);

    // We need to get rid of the newline char
    stripnl(fname);

    // Open the file. If anything else than 0 is returned there was an error
    if((fopen_s(&infile, fname, "rb")) != 0) 
    {
        printf("Error Opening File.\n");
        return(0);
    }

    // get file size
    fseek(infile, 0L, SEEK_END);
    *nLength = ftell(infile);
    // rewind
    fseek(infile, 0L, SEEK_SET);

    pInutBuf = (unsigned char *) malloc(*nLength+1);

    // Read in characters and place them in "buffer"
    ch = fgetc(infile);
    for(i=0; (i < *nLength) && (feof(infile) == 0); i++)
    {
        pInutBuf[i] = (unsigned char)ch;
        ch = fgetc(infile);
    }

    // end of buffer
    pInutBuf[i] = '\0';

    fclose(infile);
    return pInutBuf;
}


int crypt(unsigned char *key,
             unsigned char *pInputBuf,
             unsigned char *pOutputBuf,
             long nLength)
{
    int nKeyPos = 0;
    long n;
    unsigned char KeyA = 0;

    if((pInputBuf != NULL) && (pOutputBuf != NULL))
    {
        for(n = 0; n < KEY_BUF; n++)
            KeyA ^= key[n];

        nKeyPos = KeyA%KEY_BUF;

        for(n = 0; n < nLength; n++)
        {
            pOutputBuf[n] = pInputBuf[n]^(key[nKeyPos]*KeyA);
            KeyA += pOutputBuf[n];
            nKeyPos = pOutputBuf[n]%KEY_BUF;
        }
        return 0;   // success
    }
    return 1;
}

int decrypt(unsigned char *key,
     unsigned char *pOutputBuf,
     unsigned char *pInputBuf,
     long nLength)
{
     int nKeyPos = 0;
     long n;
     unsigned char KeyA = 0;

     if ((pInputBuf != NULL) && (pOutputBuf != NULL))
     {
          for (n = 0; n < KEY_BUF; n++)
               KeyA ^= key[n];

          nKeyPos = KeyA%KEY_BUF;

          for (n = 0; n < nLength; n++)
          {
               pInputBuf[n] = pOutputBuf[n] ^ (key[nKeyPos] * KeyA);
               KeyA += pInputBuf[n];
               nKeyPos = pInputBuf[n] % KEY_BUF;
          }
          return 0; // success
     }
     return 1;
}


int main(int argc, char *argv[])
{
    // KEY_BUF+1 corrected to VS2012 standard, we never use the last +1 ('\0') uchar
    unsigned char key[KEY_BUF+1]="abcdefghijklmnopqrst";
    unsigned char *pInputBuf=NULL;
    unsigned char *pOutputBuf=NULL;
    long nLength=0, n;

    if((pInputBuf = readfile(pInputBuf, &nLength)) == NULL)
        return 1;

    pOutputBuf = (unsigned char *) malloc(nLength);

    //if(crypt(key, pInputBuf, pOutputBuf, nLength) != 0)
        //return 2;

    if(decrypt(key, pOutputBuf, pInputBuf, nLength) != 0)
        return 2;

    if(writefile(pOutputBuf, nLength) == 0)
        return 3;
/*
    for(n = 0; n < nLength; n++)
        printf("%c", pInputBuf[n]);
    printf("\n");

    for(n = 0; n < nLength; n++)
        printf("%X", pOutputBuf[n]);
    printf("\n");
*/
    // Free up memory
    if(pInputBuf)
        free(pInputBuf);
    if(pOutputBuf)
        free(pOutputBuf);

    getchar(); // pause and wait for key
    return 0;
}

Answer 1

In crypt() you alter KeyA/nKeyPos based of the encrypted cipher text. In decrypt() you alter KeyA/nKeyPos based of the decrypted plain text.

Try this:

int crypt( unsigned char *key,
           const unsigned char *pPlaintext, //input
           unsigned char *pCiphertext,      //output
           long nLength)
{
    ...
    for(n = 0; n < nLength; n++)
    {
        pCiphertext[n] = pPlaintext[n] ^ (key[nKeyPos]*KeyA);
        KeyA += pCiphertext[n];
        nKeyPos = pCiphertext[n] % KEY_BUF;
    }
}

int decrypt( unsigned char *key,
             const unsigned char *pCiphertext, //input
             unsigned char *pPlaintext,        //output
             long nLength)
{
    ...
    for(n = 0; n < nLength; n++)
    {
        pPlaintext[n] = pCiphertext[n] ^ (key[nKeyPos]*KeyA);
        KeyA += pCiphertext[n];
        nKeyPos = pCiphertext[n] % KEY_BUF;
    }
}