CODEWITHSUNDEEP
c++assemblyx86encryptionxor
user2283597
user2283597

Reputation: 229

Assembly decrypting/reversing? Is XOR an issue?

I have a code function that I have attempted to reverse the effects of with no luck. The original function I have is:

          ror al,1                      // rotates the al part of the eax register (the Ekey) bitwise by 1 bit, as 1 mod 8 = 1 (al = 2D)
      ror al,1                      // moves the rightmost bit from al (the end of the Ekey) and shifts everything along
      ror al,1                      // rotates al bitwise by 1 bit, as 1 mod 8 = 1 (al = 4B)
      ror al,1                      // rotates the end 8 bits of the Ekey bitwise by 1 bit, as 1 mod 8 = 1 (al = A5)
      push ecx                      // preserves the value of the encrypted character by pushing it on the stack, the stack pointer decrements by 4 to allow this
      not eax                       // completes the ones' complement on the Ekey, toggling the bits
      mov edx,eax                   // copies the current value of the Ekey register and places it in edx, for holding
      pop eax                       // restores original register value from stack
      xor eax,edx                   // completes a bitwise exclusive or on the Ekey, with the previous value of the Ekey that was stored in edx
      ror al,1                      // rotates the last 8 bits of the Ekey bitwise by 1 bit, as 1 mod 8 = 1
      ror al,1                      // rotates al bitwise by 1 bit, as 1 mod 8 = 1
      not eax                       // completes the ones' complement on the Ekey value, 'flipping' eax entirely
      add eax,0x20                  // adds the hex value of 20 (32 in base 10) to the current value in the Ekey

I must reverse the EFFECT of the above code only, not each specific line. I have tried various things... attempt 1 (which is wrong):

      sub eax, 0x20
      not eax
      rol al, 2
      xor ecx, eax
      push eax
      mov eax, edx
      not eax
      pop ecx
      rol al, 4

My second attempt is below:

      sub eax, 0x20
      not eax
      rol al, 2 
      not eax
      xor ecx, eax

What is going wrong with this... can an xor's effect be reversed?

Upvotes: 2

Views: 1897

Answers (2)

Peter R
Peter R

Reputation: 2985

I left your function as it was but simplified the decryption:

unsigned int encrypt(unsigned int input, unsigned int key)
{
    _asm
    {
        mov ecx, input
        mov eax, key
        push ecx                      ; preserves the value of the encrypted character by pushing it on the stack, the stack pointer decrements by 4 to allow this
        not eax                       ; completes the ones' complement on the Ekey, toggling the bits
        mov edx,eax                   ; copies the current value of the Ekey register and places it in edx, for holding
        pop eax                       ; restores original register value from stack
        xor eax,edx                   ; completes a bitwise exclusive or on the Ekey, with the previous value of the Ekey that was stored in edx
        ror al,1                      ; rotates the last 8 bits of the Ekey bitwise by 1 bit, as 1 mod 8 = 1
        ror al,1                      ; rotates al bitwise by 1 bit, as 1 mod 8 = 1
        not eax                       ; completes the ones' complement on the Ekey value, 'flipping' eax entirely   
    }
}

unsigned int decrypt(unsigned int input, unsigned int key)
{
    _asm
    {
        mov eax, input 
        not eax
        rol al,1
        rol al,1
        mov edx, key
        not edx
        xor eax, edx
    }
}

int main()
{
    unsigned int data = 0xB84A35F2;
    unsigned int encrypted  = 0;
    unsigned int decrypted = 0;
    unsigned int key = 0x3DB76E8C2;

    encrypted = encrypt(data, key);
    decrypted = decrypt(encrypted, key);
    std::cout << "Original Data: " << data << "\nEncrypted Data: " << encrypted << "\nDecrypted Data: " << decrypted << "\n";
    system("PAUSE");
    return 0;
}

Upvotes: 2

Jerry Coffin
Jerry Coffin

Reputation: 490673

The obvious sequence would be something like:

; inputs:
;     edx: ekey
;     eax: "encrypted" word
; 
not eax
rol al, 1
rol al, 1
not edx
xor eax, edx

It also looks to me like the original code is unnecessarily complex. I think I'd write something more like this:

not eax
xchg eax, ecx
xor eax, ecx
rol al, 1
rol al, 1
not eax

I think more simplification may be possible as well, but I'll have to think about it to be sure.

Upvotes: 3

Related Questions