Why embedded CRC and current CRC differs?

Question

I have found this Delphi examle. It is supposed to embed CRC and check current CRC. Both should match, but I get different results. How to fix it? And how to speed it up?

CRC32Calc.pas

unit CRC32Calc;

interface

uses Classes, SysUtils, windows, messages;

type
  Long = record
    LoWord: Word;
    HiWord: Word;
  end;

const
  CRCPOLY = $EDB88320;

procedure BuildCRCTable;
function RecountCRC(b: byte; CrcOld: LongWord): LongWord;
function GetCRC32(FileName: string; Full: boolean): string;

function SetEmbeddedCRC(FileName: string): string;
function GetEmbeddedCRC(FileName: string): string;

function BytesToHexStr(pB: PByte; BufSize: LongWord): String;
function HexStrToBytes(Str: String): String;

implementation

var
  CRCTable: array [0 .. 512] Of LongWord;

  // A helper routine that creates and initializes
  // the lookup table that is used when calculating a CRC polynomial
procedure BuildCRCTable;
var
  i, j: Word;
  r: LongWord;
begin
  FillChar(CRCTable, SizeOf(CRCTable), 0);
  for i := 0 to 255 do
  begin
    r := i shl 1;
    for j := 8 downto 0 do
      if (r and 1) <> 0 then
        r := (r Shr 1) xor CRCPOLY
      else
        r := r shr 1;
    CRCTable[i] := r;
  end;
end;

// A helper routine that recalculates polynomial relative to the specified byte
function RecountCRC(b: byte; CrcOld: LongWord): LongWord;
begin
  RecountCRC := CRCTable[byte(CrcOld xor LongWord(b))
    ] xor ((CrcOld shr 8) and $00FFFFFF)
end;

// A helper routine that converts Word into String
function HextW(w: Word): string;
const
  h: array [0 .. 15] Of char = '0123456789ABCDEF';
begin
  HextW := '';
  HextW := h[Hi(w) shr 4] + h[Hi(w) and $F] + h[Lo(w) shr 4] + h[Lo(w) and $F];
end;

// A helper routine that converts LongWord into String
function HextL(l: LongWord): string;
begin
  with Long(l) do
    HextL := HextW(HiWord) + HextW(LoWord);
end;

// Calculate CRC32 checksum for the specified file
function GetCRC32(FileName: string; Full: boolean): string;
var
  f: TFileStream;
  i, CRC: LongWord;
  aBt: byte;
begin
  // Build a CRC table
  BuildCRCTable;

  CRC := $FFFFFFFF;
  // Open the file
  f := TFileStream.Create(FileName, (fmOpenRead or fmShareDenyNone));

  // To calculate CRC for the whole file use this loop boundaries
  if Full then
    for i := 0 to f.Size - 1 do
    begin
      f.Read(aBt, 1);
      CRC := RecountCRC(aBt, CRC);
    end
  else
    // To calculate CRC for the file excluding the last 4 bytes
    // use these loop boundaries
    for i := 0 to f.Size - 5 do
    begin
      f.Read(aBt, 1);
      CRC := RecountCRC(aBt, CRC);
    end;

  f.Destroy;
  CRC := Not CRC;

  Result := HextL(CRC);
end;

// Calculate CRC and writes it to the end of file
function SetEmbeddedCRC(FileName: string): string;
var
  f: TFileStream;
  CRCOffset: LongWord;
  CRC: string;
begin
  f := TFileStream.Create(FileName, (fmOpenReadWrite or fmShareDenyNone));
  CRCOffset := f.Size;

  // Append a placeholder for actual CRC to the file
  f.Seek(CRCOffset, TSeekOrigin.soBeginning);
  f.Write(PByte(HexStrToBytes('FFFFFFFF'))^, 4);

  // Obtain CRC
  CRC := GetCRC32(FileName, True);

  // Write CRC to the end of file
  f.Seek(CRCOffset, TSeekOrigin.soBeginning);
  f.Write(PByte(HexStrToBytes(CRC))^, 4);
  f.Destroy;
  Result := CRC;
end;

// Extract the CRC that was stored at last 4 bytes of a file
function GetEmbeddedCRC(FileName: string): string;
var
  f: TFileStream;
  CRCOffset: LongWord;
  pB: PByte;
begin
  GetMem(pB, 4);

  // Open file
  f := TFileStream.Create(FileName, (fmOpenRead or fmShareDenyNone));

  // Proceed upto the end of file
  CRCOffset := f.Size - 4;
  f.Seek(CRCOffset, TSeekOrigin.soBeginning);

  // Read the last four bytes where the CRC is stored
  f.Read(pB^, 4);
  f.Destroy;
  Result := BytesToHexStr(pB, 4);
end;

// A helper routine that converts byte value to string with hexadecimal integer
function BytesToHexStr(pB: PByte; BufSize: LongWord): String;
var
  i, j, b: LongWord;
begin
  SetLength(Result, 2 * BufSize);

  for i := 1 to BufSize do
  begin
    for j := 0 to 1 do
    begin
      if j = 1 then
        b := pB^ div 16
      else
        b := pB^ - (pB^ div 16) * 16;
      case b of
        0:
          Result[2 * i - j] := '0';
        1:
          Result[2 * i - j] := '1';
        2:
          Result[2 * i - j] := '2';
        3:
          Result[2 * i - j] := '3';
        4:
          Result[2 * i - j] := '4';
        5:
          Result[2 * i - j] := '5';
        6:
          Result[2 * i - j] := '6';
        7:
          Result[2 * i - j] := '7';
        8:
          Result[2 * i - j] := '8';
        9:
          Result[2 * i - j] := '9';
        10:
          Result[2 * i - j] := 'A';
        11:
          Result[2 * i - j] := 'B';
        12:
          Result[2 * i - j] := 'C';
        13:
          Result[2 * i - j] := 'D';
        14:
          Result[2 * i - j] := 'E';
        15:
          Result[2 * i - j] := 'F';
      end;
    end;

    Inc(pB);
  end;
end;

// A helper routine that converts string with hexadecimal integer to byte value
function HexStrToBytes(Str: String): String;
var
  b, b2: byte;
  lw, lw2, lw3: LongWord;
begin
  lw := Length(Str) div 2;
  SetLength(Result, lw);

  for lw2 := 1 to lw do
  begin
    b := 0;

    for lw3 := 0 to 1 do
    begin
      case Str[2 * lw2 - lw3] of
        '0':
          b2 := 0;
        '1':
          b2 := 1;
        '2':
          b2 := 2;
        '3':
          b2 := 3;
        '4':
          b2 := 4;
        '5':
          b2 := 5;
        '6':
          b2 := 6;
        '7':
          b2 := 7;
        '8':
          b2 := 8;
        '9':
          b2 := 9;
        'a':
          b2 := 10;
        'b':
          b2 := 11;
        'c':
          b2 := 12;
        'd':
          b2 := 13;
        'e':
          b2 := 14;
        'f':
          b2 := 15;
        'A':
          b2 := 10;
        'B':
          b2 := 11;
        'C':
          b2 := 12;
        'D':
          b2 := 13;
        'E':
          b2 := 14;
        'F':
          b2 := 15;
      else
        b2 := 0;
      end;

      if lw3 = 0 then
        b := b2
      else
        b := b + 16 * b2;
    end;

    Result[lw2] := char(b);
  end;
end;

end.

AppendCRC

program AppendCRC;

{$APPTYPE CONSOLE}

uses
  SysUtils, Classes,
  CRC32Calc in '..\CRC32Checker\CRC32Calc.pas';

var
  FileName: string;

begin
  { TODO -oUser -cConsole Main : Insert code here }
  if ParamCount = 1 then
  begin
    FileName := ParamStr(1);
    // Verify whether a file exists
    if not FileExists(FileName) then
    begin
      WriteLn('The specified file does not exist.');
      Exit;
    end;
    WriteLn('Full checksum (before): ' + GetCRC32(FileName, True));
    SetEmbeddedCRC(FileName);
    WriteLn('Half checksum: ' + GetCRC32(FileName, False));
    WriteLn('Full checksum (after): ' + GetCRC32(FileName, True));
    WriteLn('GetEmbeddedCRC: :' + GetEmbeddedCRC(FileName));
    WriteLn('The checksum was successfully embedded.')
  end
  else
  begin;
    WriteLn('Wrong parameters.');
    WriteLn('Parameter1 - Full path to file.');;
  end;

end.

My results are:

AppendCRC.exe Hello_Delphi_World.exe
Full checksum (before): 1912DA64
Half checksum: 1912DA64
Full checksum (after): B3F0A43E
GetEmbeddedCRC: :4400A000
The checksum was successfully embedded.

I am using Delphi XE5.

Answer 1

You should understand how this code works. Overall idea is to append the CRC as an extra 4 bytes, out of the EXE structure, to the end of file. (A better idea would be to put CRC into a special field inside EXE Header in the beginning).

However that raises the hen and the egg problem: after we calculate CRC and embed it - the CRC file is changed (the value of CRC is appended) and the CRC of changed files changes too.

So you basically has to implement two modes/function of CRC calculation: for the whole file and for the file without last 4 bytes. You should use the latter mode to calculate CRC after appending (you call it embedding), and the former one to calculate CRC before it on vanilla just compiled program.

Your GetCRC32 function always cuts last 4 bytes from the file, thus before embedding it calculates CRC only of some part of file, not of the whole file. But there ahve to be two different modes.

PS: you can also "embed" CRC into NTFS Alternate Stream, like having MyApp.exe program and CRC stored as MyApp.exe:CRC.

PPS. i think using unbuffered read byte by byte in the GetCRC32 should be very slow. If possible, better use TBytesStream to read the file into memory as whole and then scan in usual loop over array. Or read it by chunks of 4096 bytes rather than by byte variables. For the last non-complete buffer you would clean the rest of buffer with zeroes for example.