perlembed/perlcall example unbounded memory growth - valgrind says no leaks possible

Question

all. I've been banging my head up against this one for a while... I'm trying to put together an example of basically perlembed + perlcall and more or less "borrowed" it from evpsgi. The problem is that it grows about 1MB in size for every 1000 iterations. This isn't the greatest situation to be in when running in a long lived process (which is the use case that I'm working with).

As the title states, if I run with valgrind it reports that there are no leaks possible. I ran with --trace-malloc=yes and it appears that free is only ever called at the end in a big batch of calls. I understand that this may be perl's MO but it would be nice if it at least re-used the memory and didn't grow until the OS killed off the process.

The entry for sv_2mortal mentions something about the buffer being available to be "stolen" but I've peppered the code with calls to sv_2mortal but there was no change.

Without further ado, here is the code. Please forgive the cargo cultishness of it. Thanks in advance!

/*
 *
 * cc `perl -MExtUtils::Embed -e ccopts -e ldopts` -Wall -ggdb test_perl_2.c -o test_perl_2
 *
 * # test.psgi
 * use strict;
 * use warnings;
 * my $app = sub  {
 *     return [ 200, [ test => 1 ], [ sprintf( "%d: Hello!!! from %s\n", $$, __FILE__ ) ] ];
 * };
 *
 */

#include <stdio.h>
#include <EXTERN.h>               /* from the Perl distribution     */
#include <perl.h>                 /* from the Perl distribution     */

static PerlInterpreter *perlinterp;  /***    The Perl interpreter    ***/
static SV *app;

void do_stuff( void );
SV * get_stuff( void );
SV * call_stuff( SV * );

EXTERN_C void xs_init( pTHX );
EXTERN_C void boot_DynaLoader (pTHX_ CV* cv);
EXTERN_C void xs_init( pTHX ) {
    char *file = __FILE__;
    dXSUB_SYS;
    /* DynaLoader is a special case */
    newXS("DynaLoader::boot_DynaLoader", boot_DynaLoader, file);
}

int main( int argc, char **argv, char **env ) {
    char code[ 1024 ];
    char psgi[] = "test.psgi";
    char *embedding[] = { "", "-e", "0" };

    PERL_SYS_INIT3( &argc, &argv, &env );
    perlinterp = perl_alloc();
    PERL_SET_CONTEXT( perlinterp );
    perl_construct( perlinterp );
    perl_parse( perlinterp, xs_init, 3, embedding, (char **)NULL );
    PL_exit_flags |= PERL_EXIT_DESTRUCT_END;

    sprintf( code, "do '%s' or die $@", psgi );
    app = eval_pv( code, TRUE ); /* croak_on_error */

    do_stuff();

    PL_perl_destruct_level = 1;
    perl_destruct( perlinterp );
    perl_free( perlinterp );
    PERL_SYS_TERM();
    return 0;
}

void do_stuff( void ) {
    int body_lastindex, i, count;
    AV *response_av, *body_av;
    SV *stuff_sv, *response_sv, *status, *tmp_body_sv, *body_sv;

//  count = 10000;
    count = 10;
    while( count-- ) {

        ENTER;
        SAVETMPS;

        stuff_sv = get_stuff();
        response_sv = call_stuff( stuff_sv );

        if(
            NULL == response_sv ||
            ! SvROK( response_sv ) ||
            SvTYPE( SvRV( response_sv ) ) != SVt_PVAV
        ) {
            printf( "NULL == response_sv\n" );
            goto CLIENT_END;
        }

        response_av = (AV *)SvRV( response_sv );

        status = *av_fetch( response_av, 0, 0 );
        printf( "status = %ld\n", (long)SvIV( status ) );

        body_av = (AV *)SvRV( *av_fetch( response_av, 2, 0 ) );

        body_sv = newSV( 0 );

        body_lastindex = av_len( body_av );
        for( i = 0; i <= body_lastindex; i++ ) {
            tmp_body_sv = (SV *)*av_fetch( body_av, i, 0 );
            if( SvOK( tmp_body_sv ) ) {
                sv_catsv( body_sv, tmp_body_sv );
            }
        }
        printf( "body_sv = %s\n", SvPV_nolen( body_sv ) );

CLIENT_END:
        FREETMPS;
        LEAVE;
    }
}

SV * get_stuff( void ) {
    HV *stuff_hv;
//    stuff_hv = (HV *)sv_2mortal((SV *)newHV());
    stuff_hv = newHV();

    if( NULL == hv_store( stuff_hv, "SCRIPT_NAME", strlen( "SCRIPT_NAME" ), newSVpv( "", 0 ), 0 ) ) {
        croak( "hv_store( 'SCRIPT_NAME' )" );
    }

    if( NULL == hv_store( stuff_hv, "REQUEST_METHOD", strlen( "REQUEST_METHOD" ), newSVpv( "GET", 3 ), 0 ) ) {
        croak( "hv_store( 'REQUEST_METHOD' )" );
    }

    if( NULL == hv_store( stuff_hv, "REQUEST_URI", strlen( "REQUEST_URI" ), newSVpv( "/abc?def", 8 ), 0 ) ) {
        croak( "hv_store( 'REQUEST_URI' )" );
    }

    if( NULL == hv_store( stuff_hv, "PATH_INFO", strlen( "PATH_INFO" ), newSVpv( "/abc", 4 ), 0 ) ) {
        croak( "hv_store( 'PATH_INFO' )" );
    }

    if( NULL == hv_store( stuff_hv, "QUERY_STRING", strlen( "QUERY_STRING" ), newSVpv( "def", 3 ), 0 ) ) {
        croak( "hv_store( 'QUERY_STRING' )" );
    }

    return newRV_inc( (SV *)stuff_hv );
}

SV * call_stuff( SV *stuff_sv ) {
    SV *response_sv;
    int count;

//  printf( "REQUEST_URI = %s\n", SvPV_nolen( *hv_fetch( (HV *)SvRV( stuff_sv ), "REQUEST_URI", strlen( "REQUEST_URI" ), 0 ) ) );

    dSP;
    ENTER;
    SAVETMPS;
    PUSHMARK( SP );
    XPUSHs( stuff_sv ); // stuff_sv is not mortal.
    PUTBACK;
    count = call_sv( app, G_EVAL | G_SCALAR | G_KEEPERR );
    SPAGAIN;
    if( SvTRUE( ERRSV ) ) {
        response_sv = NULL;
        fprintf( stderr, "FATAL: %s", SvPV_nolen( ERRSV ) );
        /* CLEAR_ERRSV() is only available 5.8.9 or later */
        if( SvMAGICAL( ERRSV ) ) {
            mg_free( ERRSV );
            mg_clear( ERRSV );
        }
        sv_setpvn_mg( ERRSV, "", 0 );
        POPs; // causes "warning: value computed is not used"
    }
    else if( count > 0 ) {
        response_sv = POPs; // is this mortal?
        SvREFCNT_inc( response_sv );
    } else {
        response_sv = NULL;
    }

    PUTBACK;
    FREETMPS;
    LEAVE;
    return response_sv;
}

Answer 1

You don't free anything! You get a scalar from Perl and you create two yourself, but none of them are getting freed.

Leak 1

You have:

HV *stuff_hv;
stuff_hv = newHV();
return newRV_inc( (SV *)stuff_hv );

Two problems with that:

• You're creating an HV with a refcnt of 2.

  Change newRV_inc to newRV_noinc.

• You never free it (or return it from a XS func as a mortal).

  Use SvREFCNT_dec( stuff_sv ) when you're done with it, perhaps after the call to call_stuff.

Leak 2

You have:

body_sv = newSV( 0 );

Again, there's no corresponding freeing of that scalar. You need

SvREFCNT_dec( body_sv );

after the printf.

Leak 3

You have:

response_sv = POPs; // is this mortal?
SvREFCNT_inc( response_sv );

It doesn't really matter if it's mortal or not. You need to claim ownership of it in case it is, so the inc is appropriate. But you must later release it when you're done with it.

 SvREFCNT_dec( response_sv );