Adding a new system call to linux kernel doesn't seem to work

Question

I'm taking an OS course and the first assigment was to add a few system calls to the kernel (Redhat, kernel version 2.4).

I've added everything that I thought that needs to be added and it still doesn't work.

I'm trying to run tests and I noticed the wrappers (I've added an example for one below and the sys_call code for it) aren't linked the system calls. I know this because I put a printk inside the system calls and they never appear (and the test itself doesn't give the result I'm expecting).

Now from what I understand the development process for adding a system call is:

1. Create a wrappers like the example wrapper function in the instructions (lets call it aservice״).
2. Write the sys_aservice in a .c file which'll go to one of the many files in the linux-2.4.18-14custom folder. The file it goes to depends on the type of service I want to add ( e.g if it were a file system service I'd put it in fs).
3. Add the entry for the system call in the table at /usr/src/linuxlinux-2.4.18-14custom/arch/i386/kernel/ entry.S

This is what I did at first but it doesn't seem to be enough (because it doesn't seem to work).

I looked it up online (and read: How to add a system call) and I read that I need to add/change the following things inorder for the linking to work.

1. Add a line in unistd.h for the system call. It'll look something like

   define __NR_aservice

2. Add the aservice to the makefile. So for example if I put the aservice.c in the fs folder I'd add aservice.c to the makefile in fs.

3. Create a aservice.h file which'll go to linux-2.4.18-14custom/include/linux (where all the h files go apparently?). This .h file will include something called a "stub" which is automatically generated from a macro in unistd.h (which the .h file will have to include) so that a user program can use your system call. The stub's declaration looks as follows: _syscallN(return type, function name, arg1 type, arg1 name ...)" where "N" is the number of parameters.

4. Prepend to every sys_X function in aservice.c the keyword "asmlinkage" (which requires inclusion of linkage.h). So function declarations look like: asmlinkage int sys_aservice(...){..} Afterward include the .h file in the .c file.

Is all this correct? Because even after doing the following it still seems as if the system call isn't called.

Here is an example wrapper and system call implementation:

 int stop_monitor(int pid){
        long __res;
        __asm__ volatile (
                "movl $244, %%eax;" << 244 is the entry in the sys_call table in entry.S 
                "movl %1, %%ebx;"
                "int $0x80;"
                "movl %%eax, %0"
                : "=m" (__res)
                : "m" (pid)
                : "%eax", "%ebx"
        );
        if((unsigned long)(__res) >= (unsigned long)(-125)) {
            errno = -(__res);
            __res = -1;
        }
        return (int)(__res);
    }


asmlinkage int sys_stop_monitor(int pid){
    task_t* task = find_task_by_pid(pid);
    if(!task)
        return -ESRCH;
    if(task->flagMonitor == 0)
        return -EPERM;
    task->flagMonitor = 0;
    return 0;
}

What am I missing?

Answer 1

I have tested these syscall helpers that I wrote for my micro libc:

#define MKFNS(fn,...) MKFN_N(fn,##__VA_ARGS__,9,8,7,6,5,4,3,2,1,0)(__VA_ARGS__)
#define MKFN_N(fn, n0, n1, n2, n3, n4, n5, n6, n7, n8, n9, n, ...) fn##n
#define syscall(...) MKFNS(syscall,##__VA_ARGS__)

static inline long syscall0(long n){
    unsigned long r;
    asm volatile("int $128":"=a"(r):"a"(n):"memory");
    return r;
}

#define syscall1(n,a) _syscall1(n,(long)a)
static inline long _syscall1(long n, long a){
    unsigned long r;
    asm volatile("int $128":"=a"(r):"a"(n),"b"(a):"memory");
    return r;
}

#define syscall2(n,a,b) _syscall2(n,(long)a,(long)b)
static inline long _syscall2(long n, long a, long b){
    unsigned long r;
    asm volatile("int $128":"=a"(r):"a"(n),"b"(a),"c"(b):"memory");
    return r;
}

#define syscall3(n,a,b,c) _syscall3(n,(long)a,(long)b,(long)c)
static inline long _syscall3(long n, long a, long b, long c){
    unsigned long r;
    asm volatile("int $128" : "=a"(r):"a"(n),"b"(a),"c"(b),"d"(c):"memory");
    return r;
}

#define syscall4(n,a,b,c,d) _syscall4(n,(long)a,(long)b,(long)c,(long)d)
static inline long _syscall4(long n, long a, long b, long c, long d){
    unsigned long r;
    asm volatile("int $128":"=a"(r):"a"(n),"b"(a),"c"(b),"d"(c),"S"(d):"memory");
    return r;
}

#define syscall5(n,a,b,c,d,e) _syscall5(n,(long)a,(long)b,(long)c,(long)d,(long)e)
static inline long _syscall5(long n, long a, long b, long c, long d, long e){
    unsigned long __ret;
    __asm__ __volatile__ ("int $128" : "=a"(__ret) : "a"(n), "b"(a), "c"(b), "d"(c), "S"(d), "D"(e) : "memory");
    return __ret;
}

void exit(long a){asm volatile("int $128":"=a"(a):"a"(1),"b"(a):"memory");}
extern char **environ;

asm(".text\n"
".global _start\n"
"_start:\n"
"popl %ecx\n"
"movl %esp,%esi\n"
"pushl %ecx\n"
"leal 4(%esi,%ecx,4),%eax\n"
"pushl %eax\n"
"pushl %esi\n"
"pushl %ecx\n"
"movl %eax,environ\n"
"call main\n"
"pushl %eax\n"
"call exit\n"
"hlt\n"
".Lstart:\n"
".size _start,.Lstart-._start\n"
);

#define aservice(...) syscall(__NR_aservice,__VA_ARGS__)
#define write(...) syscall(__NR_write,__VA_ARGS__) //#define __NR_write 4

then you can run a simple test program to make sure you can handle syscalls

int main(void){write(1,"hello world!\n",13);}

compile with gcc -nostdlib -fno-builtin

This will let you test that you can access system calls so that you know whether it is your syscall or your way of accessing them that is the problem.