Problems with my prototype of kernel (x86_64)

Question

I'm trying to learn how a kernel works while I learn assembly, and in the duty of learning how to successfully create a bootable x86_64 kernel, I had a problem:
I tried to successfully output some text with functions in "main.c" (all the files below) by using the VGA buffer in 0xB8000, the same way that I did with the 32 bits version of the kernel prototype, but the difference is that the start files are different.
The problem here is that when I use the exact same functions for the 32 bits version, it successfully prints to the screen, but when using the new files to reach the long mode (multiboot.S and start.S) this doesn't happen, the screen just go black when testing in qemu and after a few seconds it crashes with the error message:

warning: TCG doesn't support requested feature: CPUID.01H:ECX.vmx [bit 5]
qemu-system-x86_64: Trying to execute code outside RAM or ROM at 0x00000000000a0000

Why this happens? The VGA buffer just isn't in 0xB8000, There is something wrong with the *.S files? Thanks in advance!
I will paste here the kernel files:
The kernel is composed of 4 files: "main.c", "start.S", "multiboot.S" and the linker script, "linker.ld".
This 3 files are linked and compiled without any error, the files are the following: This is main.c (you will see a "basiccolors.h", this file just define vga color codes)

#include "basiccolors.h"
#include <stddef.h>
#include <stdint.h>

volatile uint16_t* vga_buffer = (uint16_t*)0xB8000; /* memory location of the VGA textmode buffer */
/* Columns and rows of the VGA buffer */
const int VGA_COLS = 80; 
const int VGA_ROWS = 25;

/* We start displaying text in the top-left of the screen (column = 0, row = 0) */
int term_col = 0;
int term_row = 0;
uint8_t term_color = WHITE_TXT; /* This color and others are defined in basiccolors.h */

/* term_init() : This function initiates the terminal by clearing it */
void term_init()
{
    /* Clear the textmode buffer */
    for (int col = 0; col < VGA_COLS; col ++)
    {
        for (int row = 0; row < VGA_ROWS; row ++)
        {
            /* The VGA textmode buffer has size (VGA_COLS * VGA_ROWS) */
            /* Given this, we find an index into the buffer for our character */
            const size_t index = (VGA_COLS * row) + col;
            /* Entries in the VGA buffer take the binary form BBBBFFFFCCCCCCCC, where: */
            /* - B is the background color */
            /* - F is the foreground color */
            /* - C is the ASCII character */
            /*  Now we set the character to blank (a space character)   */          
        vga_buffer[index] = ((uint16_t)term_color << 8) | ' '; 
        }
    }
}

/* term_putc(char c) : This function places a single character onto the  screen */
void term_putc(char c)
{
    /* We don't want to display all characters, for example, the newline ones */
    switch (c)
    {
    case '\n': /* Newline characters should return the column to 0, and increment the row */
        {
            term_col = 0;
            term_row ++;
            break;
        }

    default: /* Normal characters just get displayed and then increment the column */
        {
        /* Like before, calculate the buffer index */           
        const size_t index = (VGA_COLS * term_row) + term_col; 
        vga_buffer[index] = ((uint16_t)term_color << 8) | c;
        term_col ++;
        break;
        }
    }

/* We need to reset the column to 0, and increment the row to get to a new line */
    if (term_col >= VGA_COLS)
    {
        term_col = 0;
        term_row ++;
    }

/* we get past the last row, so we need to reset both column and row to 0 in order to loop back to the top of the screen */
    if (term_row >= VGA_ROWS)
    {
        term_col = 0;
        term_row = 0;
    }
   }

/* term_print : prints an entire string onto the screen, remember to use the "\n" and that short of things */
void term_print(const char* str)
{
    for (size_t i = 0; str[i] != '\0'; i ++) /* Keep placing characters until we hit the null-terminating character ('\0') */
        term_putc(str[i]);
}
/* Main function of the kernel, the one that is called at the end of the loading */
void kmain(void)
{
    /* Now we should initialize the interfaces */
    term_init(); /* VGA basic interface, in "basicoutput.c/h" */
    term_print("CKA Cobalt release                                                      [0-0-1]\n");
};

This is start.S:

    .extern kmain

    .section .data

    .align 16
gdtr:
gdtr_limit:
    .word (global_descriptor_table_end - global_descriptor_table) - 1
gdtr_pointer:
    .int global_descriptor_table

    .global global_descriptor_table
global_descriptor_table:
null_descriptor:
    .quad 0x0000000000000000
code_descriptor:
    .quad 0x0020980000000000
data_descriptor:
    .quad 0x0000900000000000
global_descriptor_table_end:

    .global null_segment
    .set null_segment, (null_descriptor - global_descriptor_table)
    .global code_segment
    .set code_segment, (code_descriptor - global_descriptor_table)
    .global data_segment
    .set data_segment, (data_descriptor - global_descriptor_table)

multiboot_magic:
    .space 4
multiboot_info:
    .space 4

    .section .bss

    .global kernel_pagetable
    .align 0x1000
kernel_pagetable:
pml4:
    .space 0x1000
pdpt:
    .space 0x1000
pd:
    .space 0x1000
kernel_pagetable_end:

    .global kernel_stack
kernel_stack:
    .space 0x1000
kernel_stack_end:

    .section .text
    .code32

    .global start

start:
    cli

# store multiboot parameters in .data
    mov %eax, multiboot_magic
    mov %ebx, multiboot_info

# zerofill .bss
    cld
    mov $bss, %edi
    mov $bss_end, %ecx
    sub %edi, %ecx
    xor %eax, %eax
    rep stosb

# create pagetable for identity mapping lower 2 megabytes
# make minimal page table entries
    .set pml4_entry, (pdpt + 0x03)
    .set pdpt_entry, (pd + 0x03)
    .set pd_entry, 0b10000011
    movl $pml4_entry, pml4
    movl $pdpt_entry, pdpt
    movl $pd_entry, pd

# setup long mode
# load global descriptor table
    lgdt (gdtr)

# enable Physical Address Extension (PAE)
    mov %cr4, %eax
    bts $5, %eax
    mov %eax, %cr4

# set up page table
    mov $kernel_pagetable, %eax
    mov %eax, %cr3

# set up long mode
    .set EFER_MSR_ADDRESS, 0xC0000080
    mov $EFER_MSR_ADDRESS, %ecx
    rdmsr
    bts $8, %eax
    wrmsr

# enable paging
    mov %cr0, %eax
    bts $31, %eax
    mov %eax, %cr0

# long jump to set code segment
    ljmp $code_segment, $longmode_start

    .code64
longmode_start:
# data segment selector to all data segments
    mov $data_segment, %bx
    mov %bx, %ds
    mov %bx, %es
    mov %bx, %fs
    mov %bx, %gs

# null segment selector to ss
    mov $null_segment, %bx
    mov %bx, %ss

# set up kernel stack
    mov $kernel_stack_end, %rsp
    push $0     # debugger backtrace stops here

# call kmain
    mov multiboot_magic, %edi
    mov multiboot_info, %esi
    call kmain

# hang the computer
    cli
hang:
    hlt
    jmp hang  

This is multiboot.S:

    .set MULTIBOOT_PAGE_ALIGN, 1 << 0
    .set MULTIBOOT_MEM_INFO, 1 << 1
    .set MULTIBOOT_AOUT_KLUDGE, 1 << 16
    .set MULTIBOOT_MAGIC, 0x1BADB002
    .set MULTIBOOT_FLAGS, MULTIBOOT_PAGE_ALIGN | MULTIBOOT_MEM_INFO | MULTIBOOT_AOUT_KLUDGE
    .set MULTIBOOT_CHECKSUM, -(MULTIBOOT_MAGIC + MULTIBOOT_FLAGS)

    .section .mboot
    .align 4

    .global multiboot_header

multiboot_header:
    .int MULTIBOOT_MAGIC
    .int MULTIBOOT_FLAGS
    .int MULTIBOOT_CHECKSUM
    .int multiboot_header
    .int text
    .int data_end
    .int kernel_end
    .int start

And this is my linker.ld:

OUTPUT_FORMAT("elf64-x86-64")
OUTPUT_ARCH(i386:x86-64)
ENTRY(start)

phys = 0x0100000;

SECTIONS
{
    . = phys;
    kernel_start = .;

    .text ALIGN(4096) : AT( ADDR(.text) )
    {
        text = .;
        *(.mboot)  /* Put Multiboot header section in the beginning of .text section */
        *(.text)
        *(.rodata)
        text_end = .;
    }

    .data ALIGN(4096) : AT( ADDR(.data) )
    {
        data = .;
        *(.data)
        data_end = .;
    }

    .bss ALIGN(4096) : AT( ADDR(.bss) )
    {
        bss = .;
        *(.bss)
        bss_end = .;
    }

    kernel_end = .;
}  

All this code is compilled and linked with the following commands:
Compiling...

x86_64-elf-gcc -ffreestanding -mcmodel=large -mno-red-zone -mno-mmx -mno-sse -mno-sse2 -c <file> -o <object-file>

And linking:

x86_64-elf-gcc -ffreestanding -T linker.ld multiboot.o start.o main.o -o kernel.bin -nostdlib -lgcc  

This commands are suggested by osdev.com in the tutorial http://wiki.osdev.org/Creating_a_64-bit_kernel and all is compiled and linked using a gcc cross-compiler for the x86_64 architecture.

Answer 1

QEMU doesn't support ELF64 executables when using the -kernel parameter. You will need to boot your kernel with a multiboot2 compatible loader like GRUB2 . Unfortunately you will also need to change your multiboot header to be multiboot2 compliant. You can replace your multiboot.S file to be:

.section .mboot
.code32
.align 8

    # constants for multiboot2 header:
    .set MAGIC2,    0xe85250d6
    .set ARCH2,     0                  # i386 protected mode
    .set CHECKSUM2, (-(MAGIC2 + ARCH2 + (mboot2_end - mboot2_start)) & 0xffffffff)

     /* multiboot2 header */
mboot2_start:
    .long MAGIC2
    .long ARCH2
    .long mboot2_end - mboot2_start
    .long CHECKSUM2
    .word  0                           # type
    .word  0                           # flags
    .long  8                           # size
mboot2_end:

You compile it the same way you did before. There is a catch - in order to make sure this header doesn't end up being pushed beyond the first 8kb of the file you'll likely need to specify 4kb pages when you link:

x86_64-elf-gcc -z max-page-size=0x1000 -ffreestanding -T linker.ld \
    multiboot.o start.o main.o -o kernel.bin -nostdlib -lgcc

The addition of the -z max-page-size=0x1000 forces the maximum page size to 4kb.