Calling CreateWindowEx from x64 assembly

Question

I'm trying to create a window in NASM, and I'm encountering an issue with calling CreateWindowExW. It seems to be receiving the wrong parameters.

This is the (relevant) code for window.asm

SetupWindow:
; Stack setup
        push rbp
        mov rbp, rsp

        push r12
        push r13

; Variables
        sub rsp, sizeof(WNDCLASSEXW)

; Clear WNDCLASSEXW
        cld
        mov r12, rsp
        mov rdi, r12
        xor eax, eax
        mov ecx, sizeof(WNDCLASSEXW)
        rep stosb

; Set WNDCLASSEXW fields
        mov r13, sizeof(WNDCLASSEXW)    ; sizeof(WNDCLASSEXW) -> cbSize
        mov dword [r12 + WNDCLASSEXW.cbSize], r13d
        lea r13, [rel WindowProcedure]  ; WindowProcedure -> lpfnWndProc
        mov qword [r12 + WNDCLASSEXW.lpfnWndProc], r13
        lea r13, [rel className]        ; className -> lpszClassName
        mov qword [r12 + WNDCLASSEXW.lpszClassName], r13

; GetModuleHandleA(NULL) -> hInstance
        sub rsp, 32
        xor eax, eax
        call GetModuleHandleA
        add rsp, 32
        mov qword [r12 + WNDCLASSEXW.hInstance], rax

; RegisterClassExW
        sub rsp, 32                     ; Shadow space
        mov rcx, r12                    ; Window class structure
        call RegisterClassExW
        add rsp, 32                     ; Clear shadow space

; CreateWindowExW
        sub rsp, 80
        xor ecx, ecx                    ; No extended styles
        mov rdx, qword [r12 + WNDCLASSEXW.lpszClassName]
        lea r8, [rel windowName]        ; Window name
        mov r9, WS_OVERLAPPEDWINDOW     ; Generic window style
        mov r13, CW_USEDEFAULT          ; CW_USEDEFAULT for X, Y, nWidth, nHeight
        mov qword [rsp + 32 + 0], r13   ; X
        mov qword [rsp + 32 + 8], r13   ; Y
        mov qword [rsp + 32 + 16], r13  ; nWidth
        mov qword [rsp + 32 + 24], r13  ; nHeight
        mov qword [rsp + 32 + 32], rcx  ; hWndParent
        mov qword [rsp + 32 + 40], rcx  ; hMenu
        mov r13, qword [r12 + WNDCLASSEXW.hInstance]
        mov qword [rsp + 32 + 48], r13  ; hInstance
        mov qword [rsp + 32 + 56], rcx  ; lpParam
        call CreateWindowExW2
        add rsp, 80

; Check window
        cmp rax, 0
        jne .success

.error:
        sub rsp, 32
        call GetLastError
        add rsp, 32
        mov ecx, eax
        call ExitProcess
.success:
        mov [rel window], rax
        mov rcx, [rel window]
        mov edx, SW_SHOWNORMAL
        call ShowWindow

and this is the code for win32.inc:

; kernel32
        extern ExitProcess
        extern GetLastError
        extern GetModuleHandleA

; user32
        extern CreateWindowExW
        extern RegisterClassExW
        extern ShowWindow

        struc WNDCLASSEXW
            .cbSize: resd 1
            .style: resd 1
            .lpfnWndProc: resq 1
            .cbClsExtra: resd 1
            .cbWndExtra: resd 1
            .hInstance: resq 1
            .hIcon: resq 1
            .hCursor: resq 1
            .hbrBackground: resq 1
            .lpszMenuName: resq 1
            .lpszClassName: resq 1
            .hIconSm: resq 1
        endstruc

        %define CW_USEDEFAULT 80000000h

        %define SW_SHOWNORMAL 1

        %define WS_CAPTION 00C00000h
        %define WS_MAXIMIZEBOX 00010000h
        %define WS_MINIMIZEBOX 00020000h
        %define WS_OVERLAPPED 00000000h
        %define WS_SYSMENU 00080000h
        %define WS_THICKFRAME 00040000h

        %define WS_OVERLAPPEDWINDOW (WS_CAPTION | WS_MAXIMIZEBOX | WS_MINIMIZEBOX | \
                WS_OVERLAPPED | WS_SYSMENU | WS_THICKFRAME)

The full code is on GitHub: https://github.com/MobSlicer152/pong

Answer 1

I figured out what I did wrong. Basically, I thought after the four register parameters, it was the same as the X86 calling convention. However, it isn't. Integer parameters go in left-to-right order, and are padded to 8 bytes. Making this change seems to have fixed the problem I had.

However, then the function started returning NULL, but GetLastError would return 0 (ERROR_SUCCESS).

I solved that. I realised that CreateWindow fails if the window procedure returns false when handling the WM_NCCREATE message, which it receives during CreateWindow. I changed it to return true, and it succeeds now.