Writing a function in 64 bit assembly

Question

I am trying to write a function (max) in 64 bit assembly and I don't know what I am doing wrong, maybe some of you guys can determine what I am doing wrong :/

Here's the function:

int max(int a, int b) {
/* Return the larger of the two integers `a' and `b'. */
if (a > b)
return a;
else
return b;
}

And here is my assembly code (with comments):

    push %rbp
    mov %rsp, %rbp
    mov %rdi, %rax
    mov %rsi, %rcx
    test %rax, %rax                // Checking if first parameter is signed
    js .signedRAX
    test %rcx, %rcx                // Checking if second parameter is signed
    js .signedRCX
    jmp .compare                   // If either one signed then jump to .compare
  .signedRAX:
    test %rcx, %rcx                // Checking if both are signed
    js .signedRAXandRCX
    mov %rcx, %rax                 // If not then return the positive number
    jmp .end                       // finish the function
  .signedRCX:
    jmp .end                       // If only the second parameter is signed then jump  
  .signedRAXandRCX:                // straight to end of function and return %rax
    cmp %rax, %rcx                 // If both are signed compare which one is the max
    jl .end 
    mov %rcx, %rax
    jmp .end
  .compare:
    cmp %rax, %rcx                 // If both are positive then compare which one is 
    jg .end                        // the max
    mov %rcx, %rax
  .end:
    mov %rbp, %rsp
    pop %rbp
    ret

I am getting the wrong output when comparing two parameters that are both signed and then both positive.

Answer 1

Here is equivalent C-pseudo code for ASM from question. You can see, for a >= 0 and b < 0 it return b. For a < 0 and b >= 0 it return a. That's incorrect. There may be another errors in the code, because so simple operation encoded really messy. It's really hard to see anything in the code. Don't make simple things so complex. Follow KISS principle.

// test %rax, %rax
// js .signedRAX
if (a >= 0) {
    // test %rcx, %rcx
    // js .signedRCX
    if (b >= 0) {
        // .compare
        // cmp %rax, %rcx                 // If both are positive then compare which one is 
        // jg .end                        // the max
        if (a > b) {
            b = a;
        }
        return b;
    } else {
        // .signedRCX
        return b;
    }
} else {
    // .signedRAX
    // test %rcx, %rcx                // Checking if both are signed
    // js .signedRAXandRCX
    if (b >= 0) {
        b = a;
        return b;
    } else {
        // .signedRAXandRCX
        // cmp %rax, %rcx                 // If both are signed compare which one is the max
        // jl .end
        if (a < b) {
            b = a;
        }
        return b;
    }
}

Answer 2

You work nuch too complicated.

If I input your program to gcc -S, I get

max:
.LFB0:
        .cfi_startproc
        pushl   %ebp
        .cfi_def_cfa_offset 8
        .cfi_offset 5, -8
        movl    %esp, %ebp
        .cfi_def_cfa_register 5
        movl    8(%ebp), %eax
        cmpl    12(%ebp), %eax
        jle     .L2
        movl    8(%ebp), %eax
        jmp     .L3
.L2:
        movl    12(%ebp), %eax
.L3:
        popl    %ebp
        .cfi_restore 5
        .cfi_def_cfa 4, 4
        ret
        .cfi_endproc
.LFE0:

If I take over your "ABI" and way of passing arguments,

I get

max:
    push %rbp
    mov %rsp, %rbp
    mov %rdi, %rax
    mov %rsi, %rcx
    cmpl %rcx, %rax
    jle .L2
    movl %rcx, %rax
.L2:
    mov %rbp, %rsp
    pop %rbp
    ret