Visual Studio LNK1107: invalid or corrupt file - .obj (COFF) from Flat Assembler

Question

I am trying to compile somebody's program that is partly written in assembler and partly in C language. The program is available in compiled version, however I need to change it a bit. The source code of the program consists of one .c file and one .asm file. Without changing anything yet I tried to compile the program in Visual Studio 2017 and Flat Assembler (version 1.73). I compiled the .asm file and Flat Assembler gave me .obj file (unlzws4.obj). Then I created a new empty project in VS and imported .c file and .obj file from FASM.

Here the problem appears. During the compilation Visual Studio gives me this linking error:

1>------ Rebuild All started: Project: unlibs4, Configuration: Debug Win32 ------
1>unlibs4.c
1>unlzws4.obj : fatal error LNK1107: invalid or corrupt file: cannot read at 0x10EE4
1>Done building project "unlibs4.vcxproj" -- FAILED.
========== Rebuild All: 0 succeeded, 1 failed, 0 skipped ==========

I tried to find out why this error appears, however with no results. The code is supposed to be correct.

unlibs4.c file:

// (c) CTPAX-X Team 2018 http://www.CTPAX-X.org/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <malloc.h>
#include <windows.h>

#ifdef GCC32HACK
#include "win32gcc.h"
#endif

#define S4L_REVISION 0x1000
#define S4F_PACKED   1
#define S4F_CRYPTED  2
#pragma pack(push, 1)
typedef struct
{
    DWORD headsize; /* size of this header with all data */
    DWORD revision; /* MUST be equal to 0x1000 */
    DWORD dnamelen; /* directory names block len */
    DWORD dcounter; /* directory count in block  */
    DWORD fnamelen; /* file names block len */
    DWORD fcounter; /* files counter */
} headinfo;

typedef struct
{
    DWORD floffs; /* looks like first always 1 (.LIB file starts with one zero byte) */
    DWORD pksize; /* packed size */
    DWORD unsize; /* unpacked size (zero if file not packed) */
    DWORD dindex; /* directory name index */
    DWORD xflags; /* flags: 1 - packed; 2 - crypted; */
    DWORD crcsum; /* file CRC (16 bit); can be zero; only for packed files? */
} fileinfo;
#pragma pack(pop)


extern void WINAPI unlzws4(BYTE *undata, DWORD *unsize, BYTE *pkdata, DWORD *pksize);

int main(int argc, char *argv[])
{
    char *dnames, *fnames, *s, *d, name[MAX_PATH];
    fileinfo *list;
    FILE *fl, *f;
    headinfo hi;
    DWORD i, j;
    BYTE *p, *u;
    printf("The Settlers IV .LIB unpacker v1.0\n(c) CTPAX-X Team 2018\nhttp://www.CTPAX-X.org/\n\n");
    if (argc != 2)
    {
        printf("Usage: unlibs4 <filename.lib>\n\n");
        return(1);
    }
    fl = fopen(argv[1], "rb");
    if (!fl)
    {
        printf("Error: can't open input file.\n\n");
        return(2);
    }
    /* read TOC offs */
    fseek(fl, 0, SEEK_END);
    fseek(fl, ftell(fl) - 4, SEEK_SET);
    fread(&i, 4, 1, fl);
    /* read TOC */
    fseek(fl, i, SEEK_SET);
    memset(&hi, 0, sizeof(hi));
    fread(&hi, sizeof(hi), 1, fl);
    if (hi.revision != S4L_REVISION)
    {
        fclose(fl);
        printf("Error: invalid/unknown input file format.\n\n");
        return(3);
    }
    /* allocate memory */
    dnames = (char *)malloc(hi.dnamelen);
    fnames = (char *)malloc(hi.fnamelen);
    list = (fileinfo *)malloc(hi.fcounter * sizeof(list[0]));
    if (dnames && fnames && list)
    {
        /* read strings */
        fread(dnames, hi.dnamelen, 1, fl);
        fread(fnames, hi.fnamelen, 1, fl);
        /* read file infos */
        fread(list, hi.fcounter, sizeof(list[0]), fl);
        /* create all directories before */
        d = dnames;
        for (i = 0; i < hi.dcounter; i++)
        {
            CreateDirectory(d, NULL);
            d += strlen(d) + 1;
        }
        /* unpack */
        s = fnames;
        for (i = 0; i < hi.fcounter; i++)
        {
            /* find directory name */
            d = dnames;
            j = list[i].dindex;
            while (j > 0)
            {
                d += strlen(d) + 1;
                j--;
            }
            /* merge directory with file */
            sprintf(name, "%s\\%s", d, s);
            /* dump to disk */
            printf("%s\n", name);
            f = fopen(name, "wb");
            if (f)
            {
                p = (BYTE *)malloc(list[i].pksize);
                if (p)
                {
                    fseek(fl, list[i].floffs, SEEK_SET);
                    fread(p, list[i].pksize, 1, fl);
                    if (list[i].xflags & S4F_CRYPTED)
                    {
                        printf("Warning: encrypted file, but decryption not supported - saving as is.\n");
                    }
                    /* decryption not supported, that's why '==' instead of just '&' */
                    if ((list[i].xflags & (S4F_CRYPTED | S4F_PACKED)) == S4F_PACKED)
                    {
                        u = (BYTE *)malloc(list[i].unsize);
                        if (u)
                        {
                            j = list[i].unsize;
                            unlzws4(u, &list[i].unsize, p, &list[i].pksize);
                            free(p);
                            p = u;
                            list[i].pksize = j;
                        }
                    }
                    fwrite(p, list[i].pksize, 1, f);
                    free(p);
                }
                fclose(f);
            }
            /* next filename */
            s += strlen(s) + 1;
        }
        printf("\ndone\n\n");
    }
    if (dnames) { free(dnames); }
    if (fnames) { free(fnames); }
    if (list) { free(list); }
    fclose(fl);
    return(0);
}

Beginning of the unlzws4.asm file:

; To compile this source code you'll need FASM:
; http://flatassembler.net/
FORMAT COFF

include 'win32ax.inc'

; define section so this can be merged to the code block
section '.text' code

public @startup as '_unlzws4@16'

; some consts
;dword_616478 - start
dword_6167E0 = buf_all + 6167E0h - 616478h
dword_6167E4 = buf_all + 6167E4h - 616478h
dword_616DF8 = buf_all + 616DF8h - 616478h
dword_617060 = buf_all + 617060h - 616478h
dword_617064 = buf_all + 617064h - 616478h

; ###########################################################################

; some startup initialization code
@startup:
  push    esi
  push    edi
  xor     eax, eax
  cld
  ; clear all buffers
  mov     edi, buf_65536
  mov     ecx, 65536 / 4
  rep stosd
  ; ---
  mov     edi, buf_548
  mov     ecx, 548 / 4
  rep     stosd
  ; reinit static buffer since it can be modified due unpacking (sic!)
  mov     esi, dword_616478
  mov     edi, buf_all
  mov     ecx, (3112d + 152d) / 4
  rep     movsd
  ; ---
  pop     edi
  pop     esi
  ; emulate some class structure
  mov     ecx, _buf
  ; jump to the actual unpacking routine
  jmp     @sub_573090

; ###########################################################################

@sub_572450:
var_4           = -4
arg_0           =  4
arg_4           =  8
  push    ecx
  push    ebx
  mov     ebx, [esp+8+arg_0]
  push    ebp
  mov     ebp, [esp+0Ch+arg_4]
  push    esi
  push    edi
  mov     esi, 28h
@loc_572462:

...

I don't know much about assembler, I just need to change the C part of the program and compile it as .dll library. I am wondering if the error appears just because of some settings in Visual Studio project. Everything is set to default options.

Answer 1

To link using the Microsoft linker you need to have MSCOFF format object files. Where you have

FORMAT COFF

You need:

FORMAT MS COFF

From https://flatassembler.net/docs.php?article=manual#2.4.3 :

2.4.3 Common Object File Format

To select Common Object File Format, use format COFF or format MS COFF directive, depending whether you want to create classic (DJGPP) or Microsoft's variant of COFF file. The default code setting for this format is 32-bit. To create the file in Microsoft's COFF format for the x86-64 architecture, use format MS64 COFF setting, in such case long mode code is generated by default.