How do I prevent a C shared library to print on stdout in python?

Question

I work with a python lib that imports a C shared library that prints on stdout. I want a clean output in order to use it with pipes or to redirect in files. The prints are done outside of python, in the shared library.

At the beginning, my approach was:

# file: test.py
import os
from ctypes import *
from tempfile import mktemp

libc = CDLL("libc.so.6")

print # That's here on purpose, otherwise hello word is always printed

tempfile = open(mktemp(),'w')
savestdout = os.dup(1)
os.close(1)
if os.dup(tempfile.fileno()) != 1:
    assert False, "couldn't redirect stdout - dup() error"

# let's pretend this is a call to my library
libc.printf("hello world\n")

os.close(1)
os.dup(savestdout)
os.close(savestdout)

This first approach is half working:
- For some reason, it needs a "print" statement just before moving stdout, otherwise hello word is always printed. As a result it will print an empty line instead of all the fuzz the library usually outputs.
- More annoying, it fails when redirecting to a file:

$python test.py > foo && cat foo

hello world

My second python attempt was inspired from another similar thread given in the comments:

import os
import sys
from ctypes import *
libc = CDLL("libc.so.6")

devnull = open('/dev/null', 'w')
oldstdout = os.dup(sys.stdout.fileno())
os.dup2(devnull.fileno(), 1)

# We still pretend this is a call to my library
libc.printf("hello\n")

os.dup2(oldstdout, 1)

This one also fails to prevent "hello" from printing.

Since I felt this was a bit low level, I then decided to go completely with ctypes. I took inspiration from this C program, which does not print anything:

#include <stdio.h>

int main(int argc, const char *argv[]) {
    char buf[20];
    int saved_stdout = dup(1);
    freopen("/dev/null", "w", stdout);

    printf("hello\n"); // not printed

    sprintf(buf, "/dev/fd/%d", saved_stdout);
    freopen(buf, "w", stdout);

    return 0;
}

I built the following example:

from ctypes import *
libc = CDLL("libc.so.6")

saved_stdout = libc.dup(1)
stdout = libc.fdopen(1, "w")
libc.freopen("/dev/null", "w", stdout);

libc.printf("hello\n")

libc.freopen("/dev/fd/" + str(saved_stdout), "w", stdout)

This prints "hello", even if I libc.fflush(stdout) just after the printf. I am starting to think it may be not possible to do what I want in python. Or maybe the way I get a file pointer to stdout is not right.

What do you think?

Answer 1

All previous answers did not work for me, either resulting in a Segmentation Fault or not closing the outputs properly. Copying this answer and adding an additional close statement looked like it does not seem leak any file descriptors and worked in my case:

class HideOutput(object):
    '''
    A context manager that block stdout for its scope, usage:

    with HideOutput():
        os.system('ls -l')
    '''

    def __init__(self, *args, **kw):
        sys.stdout.flush()
        self._origstdout = sys.stdout
        self._oldstdout_fno = os.dup(sys.stdout.fileno())
        self._devnull = os.open(os.devnull, os.O_WRONLY)

    def __enter__(self):
        self._newstdout = os.dup(1)
        os.dup2(self._devnull, 1)
        os.close(self._devnull)
        sys.stdout = os.fdopen(self._newstdout, 'w')

    def __exit__(self, exc_type, exc_val, exc_tb):
        sys.stdout = self._origstdout
        sys.stdout.flush()
        os.dup2(self._oldstdout_fno, 1)
        os.close(self._oldstdout_fno) # Additional close to not leak fd

99% copied from this and closing the file as mentioned in this answer.

Answer 2

jfs's answer gives me an error so I came up with another solution based on this answer.

ValueError: I/O operation on closed file.

import contextlib

@contextlib.contextmanager
def silence_stderr():
    stderr_fd = sys.stderr.fileno()
    orig_fd = os.dup(stderr_fd)
    null_fd = os.open(os.devnull, os.O_WRONLY)
    os.dup2(null_fd, stderr_fd)
    try:
        yield
    finally:
        os.dup2(orig_fd, stderr_fd)
        os.close(orig_fd)
        os.close(null_fd)

Usage is quite simple, as expected.

with silence_stderr():
    # call python module: stderr will be silenced
    # call c/c++ library: stderr will be silenced

You can easily modify the code to silence stdout instead of stderr by a simple find-replace.

Answer 3

The top answer here is very good. However, it requires sys.stdout.close() which conflicts with Juypter, if one is using Python notebooks. There is an awesome project called Wurlitzer that solves the underlying problem with a context manager as well as being not only usable in Jupter, but also provides a native Jupyer extension.

https://github.com/minrk/wurlitzer

https://pypi.org/project/wurlitzer/

pip install wurlitzer

from wurlitzer import pipes

with pipes() as (out, err):
    call_some_c_function()

stdout = out.read()

from io import StringIO
from wurlitzer import pipes, STDOUT

out = StringIO()
with pipes(stdout=out, stderr=STDOUT):
    call_some_c_function()

stdout = out.getvalue()

from wurlitzer import sys_pipes

with sys_pipes():
    call_some_c_function()

And the most magical part: it supports Jupyter:

%load_ext wurlitzer

Answer 4

Based on @Yinon Ehrlich's answer. This variant tries to avoid leaking file descriptors:

import os
import sys
from contextlib import contextmanager

@contextmanager
def stdout_redirected(to=os.devnull):
    '''
    import os

    with stdout_redirected(to=filename):
        print("from Python")
        os.system("echo non-Python applications are also supported")
    '''
    fd = sys.stdout.fileno()

    ##### assert that Python and C stdio write using the same file descriptor
    ####assert libc.fileno(ctypes.c_void_p.in_dll(libc, "stdout")) == fd == 1

    def _redirect_stdout(to):
        sys.stdout.close() # + implicit flush()
        os.dup2(to.fileno(), fd) # fd writes to 'to' file
        sys.stdout = os.fdopen(fd, 'w') # Python writes to fd

    with os.fdopen(os.dup(fd), 'w') as old_stdout:
        with open(to, 'w') as file:
            _redirect_stdout(to=file)
        try:
            yield # allow code to be run with the redirected stdout
        finally:
            _redirect_stdout(to=old_stdout) # restore stdout.
                                            # buffering and flags such as
                                            # CLOEXEC may be different

Answer 5

Combining both answers - https://stackoverflow.com/a/5103455/1820106 & https://stackoverflow.com/a/4178672/1820106 to context manager that blocks print to stdout only for its scope (the code in the first answer blocked any external output, the latter answer missed the sys.stdout.flush() at end):

class HideOutput(object):
    '''
    A context manager that block stdout for its scope, usage:

    with HideOutput():
        os.system('ls -l')
    '''

    def __init__(self, *args, **kw):
        sys.stdout.flush()
        self._origstdout = sys.stdout
        self._oldstdout_fno = os.dup(sys.stdout.fileno())
        self._devnull = os.open(os.devnull, os.O_WRONLY)

    def __enter__(self):
        self._newstdout = os.dup(1)
        os.dup2(self._devnull, 1)
        os.close(self._devnull)
        sys.stdout = os.fdopen(self._newstdout, 'w')

    def __exit__(self, exc_type, exc_val, exc_tb):
        sys.stdout = self._origstdout
        sys.stdout.flush()
        os.dup2(self._oldstdout_fno, 1)

Answer 6

Yeah, you really want to use os.dup2 instead of os.dup, like your second idea. Your code looks somewhat roundabout. Don't muck about with /dev entries except for /dev/null, it's unnecessary. It's also unnecessary to write anything in C here.

The trick is to save the stdout fdes using dup, then pass it to fdopen to make the new sys.stdout Python object. Meanwhile, open an fdes to /dev/null and use dup2 to overwrite the existing stdout fdes. Then close the old fdes to /dev/null. The call to dup2 is necessary because we can't tell open which fdes we want it to return, dup2 is really the only way to do that.

Edit: And if you're redirecting to a file, then stdout is not line-buffered, so you have to flush it. You can do that from Python and it will interoperate with C correctly. Of course, if you call this function before you ever write anything to stdout, then it doesn't matter.

Here is an example that I just tested that works on my system.

import zook
import os
import sys

def redirect_stdout():
    print "Redirecting stdout"
    sys.stdout.flush() # <--- important when redirecting to files
    newstdout = os.dup(1)
    devnull = os.open(os.devnull, os.O_WRONLY)
    os.dup2(devnull, 1)
    os.close(devnull)
    sys.stdout = os.fdopen(newstdout, 'w')

zook.myfunc()
redirect_stdout()
zook.myfunc()
print "But python can still print to stdout..."

The "zook" module is a very simple library in C.

#include <Python.h>
#include <stdio.h>

static PyObject *
myfunc(PyObject *self, PyObject *args)
{
    puts("myfunc called");
    Py_INCREF(Py_None);
    return Py_None;
}

static PyMethodDef zookMethods[] = {
    {"myfunc",  myfunc, METH_VARARGS, "Print a string."},
    {NULL, NULL, 0, NULL}
};

PyMODINIT_FUNC
initzook(void)
{
    (void)Py_InitModule("zook", zookMethods);
}

And the output?

$ python2.5 test.py
myfunc called
Redirecting stdout
But python can still print to stdout...

And redirecting to files?

$ python2.5 test.py > test.txt
$ cat test.txt
myfunc called
Redirecting stdout
But python can still print to stdout...

Answer 7

Here is how I finally did. I hope this can be useful for other people (this works on my linux station).

I proudly present the libshutup, designed for making external libraries shut up.

1) Copy the following file

// file: shutup.c
#include <stdio.h>
#include <unistd.h>

static char buf[20];
static int saved_stdout;

void stdout_off() {
    saved_stdout = dup(1);
    freopen("/dev/null", "w", stdout);
}

void stdout_on() {
    sprintf(buf, "/dev/fd/%d", saved_stdout);
    freopen(buf, "w", stdout);
}

2) Compile it as a shared library

gcc -Wall -shared shutup.c -fPIC -o libshutup.so

3) Use it in you code like this

from ctypes import *
shutup = CDLL("libshutup.so")

shutup.stdout_off()

# Let's pretend this printf comes from the external lib
libc = CDLL("libc.so.6")
libc.printf("hello\n")

shutup.stdout_on()

Answer 8

Wouldn't you be able to do this the same as you would in Python? You'd import sys and point sys.stdout and sys.stderr to something that isn't the default sys.stdout and sys.stderr? I do this all the time in a few apps where I have to slurp up output from a library.