Eduardo J. Sanchez
Reputation: 776
C/Fortran hybrid code using MPI/ScaLAPACK seg faults in a weird way
I am trying to translate this CODE from Fortran into C.
This is what I have so far:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "mpi.h"
#define PRINT_NOTHING 0
#define PRINT_UP_TO_MESGS 1
#define PRINT_UP_TO_ARRAYS 2
#define PRINT_UP_TO_MATRICES 3
#define PRINT_LEVEL PRINT_UP_TO_MATRICES
/*! Parameters: */
#define TOTMEM 425053208
#define INTMEM 13107200
#define NTESTS 20
#define DLEN_ 9
#define DBLESZ 8 /*! Size of a DOUBLE PRE. */
#define MEMSIZ 425053208/8 /*! Memory for DOUBLE PRE. */
#define MASTER 0
#define MPI_ERROR_CODE_ERROR_GRID 1
#define MPI_ERROR_CODE_NOT_FACT 2
#define MPI_ERROR_CODE_NOT_SOLVE 3
#define BLACS_USER_WONT_FIN_MPI 0
#define BLACS_USER_WILL_FIN_MPI 1
#define DESCRIPTOR_SIZE 7
typedef enum {
FALSE,
TRUE
} logical;
/*! External subroutines: */
extern void Cblacs_barrier (int context,
char* scope);
extern void Cblacs_exit (int doneflag);
extern void Cblacs_get (int ,
int ,
int *context);
extern void Cblacs_gridexit (int context);
extern void Cblacs_gridinfo (int context,
int *our_nprow,
int *our_npcol,
int *my_prow,
int *my_pcol);
extern void Cblacs_gridinit (int* context,
char* order,
int nproc_rows,
int nproc_cols);
extern void Cblacs_pinfo (int *my_blacs_pid,
int *our_blacs_nprocs);
/* http://www.netlib.org/scalapack/explore-html/dd/d22/descinit_8f.html */
extern void descinit_ (int *desc,
int *m,
int *n,
int *mb,
int *nb,
int *irsrc,
int *icsrc,
int *ictxt,
int *lld,
int *info);
/* http://www.netlib.org/scalapack/explore-html/dc/d44/igsum2d___8c.html */
extern void igsum2d_ (int *contxt,
char *scope,
char *top,
int *m,
int *n,
int *a,
int *lda,
int *rdest,
int *cdest);
/* http://www.netlib.org/scalapack/explore-html/db/da1/pdbmatgen_8f.html */
extern void pdbmatgen_ (int *ictxt,
char *aform,
char *aform2,
int *bwl,
int *bwu,
int *n,
int *mb,
int *nb,
double *a,
int *lda,
int *iarow,
int *iacol,
int *iseed,
int *myrow,
int *mycol,
int *nprow,
int *npcol);
/* http://www.netlib.org/scalapack/explore-html/d8/dba/pdchekpad_8f.html */
extern void pdchekpad_ (int * ictxt,
char *mess,
int *m,
int *n,
double *a,
int *lda,
int *ipre,
int *ipost,
double *chkval);
/* http://www.netlib.org/scalapack/explore-html/d2/dc2/pddblaschk_8f.html */
extern void pddblaschk_ (char *symm,
char *uplo,
char *trans,
int *n,
int *bwl,
int *bwu,
int *nrhs,
double *x,
int *ix,
int *jx,
int *descx,
int *iaseed,
double *a,
int *ia,
int *ja,
int *desca,
int *ibseed,
double *anorm,
double *resid,
double *work,
int *worksiz);
/* http://www.netlib.org/scalapack/explore-html/d6/d3b/pdfillpad_8f.html */
extern void pdfillpad_ (int *ictxt,
int *m,
int *n,
double *a,
int *lda,
int *ipre,
int *ipost,
double *chkval);
/* http://www.netlib.org/scalapack/explore-html/d0/d2b/pdgbinfo_8f.html */
extern void pdgbinfo_ (char *summry,
int *nout,
char *trans,
int *nmat,
int *nval,
int *ldnval,
int *nbw,
int *bwlval,
int *bwuval,
int *ldbwval,
int *nnb,
int *nbval,
int *ldnbval,
int *nnr,
int *nrval,
int *ldnrval,
int *nnbr,
int *nbrval,
int *ldnbrval,
int *ngrids,
int *pval,
int *ldpval,
int *qval,
int *ldqval,
float *thresh,
double *work, /* Array to gather and bcast. */
int *iam,
int *nprocs);
/* http://www.netlib.org/scalapack/explore-html/dd/dad/pdgbtrf_8f.html */
extern void pdgbtrf_ (int *n,
int *bwl,
int *bwu,
double *a,
int *ja,
int *desca,
int *ipiv,
double *af,
int *laf,
double *work,
int *lwork,
int *info);
/* http://www.netlib.org/scalapack/explore-html/d9/dda/pdgbtrs_8f.html */
extern void pdgbtrs_ (char *trans,
int *n,
int *bwl,
int *bwu,
int *nrhs,
double *a,
int *ja,
int *desca,
int *ipiv,
double *b,
int *ib,
int *descb,
double *af,
int *laf,
double *work,
int *lwork,
int *info);
/*
http://www.netlib.org/scalapack/explore-html/d2/d66/_e_i_g_2pdmatgen_8f.html
*/
extern void pdmatgen_ (int *ictxt,
char *aform,
char *diag,
int *m,
int *n,
int *mb,
int *nb,
double *a,
int *lda,
int *iarow,
int *iacol,
int *iseed,
int *iroff,
int *irnum,
int *icoff,
int *icnum,
int *myrow,
int *mycol,
int *nprow,
int *npcol);
/* http://www.netlib.org/scalapack/explore-html/d2/dcd/sltimer_8f_source.html */
extern void slboot_ (void);
extern void slcombine_ (int *ictxt,
char *scope,
char *op,
char *timetype,
int *n,
int *ibeg,
double *times);
extern void sltimer_ ( int *i );
/*! External functions: */
/* http://www.netlib.org/scalapack/explore-html/d4/d48/numroc_8f.html */
extern int numroc_ (int *n,
int *nb,
int *iproc,
int *isrcproc,
int *nprocs);
/* http://www.netlib.org/scalapack/explore-html/df/dee/tools_8f.html#a67ae4efe5110e3297b1e9e3a46d8c78b */
extern logical lsame_ (char *ca,
char *cb);
/* http://www.netlib.org/scalapack/explore-html/db/dd0/pdlange_8f.html */
extern double pdlange_ (char *norm,
int *m,
int *n,
double *a,
int *ia,
int *ja,
int *desca,
double *work );
/*! Intrinsic functions: */
/* INTRINSIC DBLE, MAX, MIN, MOD */
inline double DBLE(int xx) { return (double) xx; }
inline double MAX(double xx, double yy) { return xx >= yy? xx: yy; }
inline double MIN(double xx, double yy) { return xx <= yy? xx: yy; }
inline int MOD(int xx, int yy) { return xx%yy; }
/*! PROGRAM PDGBDRIVER: */
int main (int argc, char **argv) {
/*! Parameters: */
int ntests = NTESTS; /* Number of num. tests to be performed */
// int BLOCK_CYCLIC_2D = 1;
// int DTYPE_ = 1;
// int CTXT_ = 2;
// int M_ = 3;
// int N_ = 4;
// int MB_ = 5;
// int NB_ = 6;
// int RSRC_ = 7;
// int CSRC_ = 8;
// int LLD_ = 9;
//
// double ZERO = 0.0;
// double PADVAL = -9923.0;
//
// int INT_ONE = 1;
/*! Local scalars: */
logical CHECK; /* Should or shouldn't I perform the num. tests? */
char TRANS; /* Should I transpose the matrix? */
// char PASSED[6];
char OUTFILE[80]; /* ? */
// int BWL;
// int BWU;
// int BW_NUM;
// int FILLIN_SIZE;
// int FREE_PTR;
// int H;
// int HH;
int I; /* Iterator per process grid. */
int IAM; /* My MPI process ID. */
int IASEED; /* Seed for random matrix A creation. */
int IBSEED; /* Seed for random matrix B creation. */
// int ICTXT;
// int ICTXTB;
// int IERR_TEMP;
// int IMIDPAD;
// int INFO;
// int IPA;
// int IPB;
// int IPOSTPAD;
// int IPREPAD;
// int IPW;
// int IPW_SIZE;
// int IPW_SOLVE;
// int IPW_SOLVE_SIZE;
// int IP_DRIVER_W;
// int IP_FILLIN;
// int J;
// int K;
/*! Data statements: */
// int KFAIL = 4;
// int KPASS = 4;
// int KSKIP = 4;
// int KTESTS = 4;
/*! Local scalars: (continued) */
// int MYCOL;
// int MYRHS_SIZE;
// int MYROW;
// int N;
// int NB;
int NBW; /* Number of bandwith values per matrix. */
int NGRIDS; /* Number of process grids to try for. */
int NMAT; /* Number of matrices */
int NNB; /* Number of sizes of NB (block column size). */
int NNBR;
int NNR;
int NOUT; /* Device out... o.O */
// int NP;
int NPCOL; /* Procs per row in processors grid/ */
int NPROCS; /* The size of OUR communicator. */
// int NPROCS_REAL;
int NPROW; /* Procs per row in processors grid/ */
// int NQ;
// int NRHS;
// int N_FIRST;
// int N_LAST;
// int WORKSIZ;
float THRESH; /* Threshold variable for numerical tests. */
// double ANORM;
// double NOPS;
// double NOPS2;
// double SRESID;
// double TMFLOPS;
// double TMFLOPS2;
/*! Local arrays: */
// int IPIV[INTMEM];
int BWLVAL[NTESTS]; /* Values of lower diagonals per matrix. */
int BWUVAL[NTESTS]; /* Values of upper diagonals per matrix. */
// int DESCA[7];
// int DESCA2D[DLEN_];
// int DESCB[7];
// int DESCB2D[DLEN_];
// int IERR[1];
int NBRVAL[NTESTS]; /* ? */
int NBVAL[NTESTS]; /* Column sizes per matrix. */
int NRVAL[NTESTS]; /* ? */
int NVAL[NTESTS]; /* Values of N for a given matrix. */
int PVAL[NTESTS]; /* Values for proc. rows. */
int QVAL[NTESTS]; /* Values for proc. cols. */
// double CTIME[2];
double *MEM; /* GLOABL array for broadcasting the information. */
// double WTIME[2];
/*! Executable statements: */
/*! Get starting information: */
Cblacs_pinfo(&IAM, &NPROCS);
IASEED = 100;
IBSEED = 200;
MEM = (double *) malloc(MEMSIZ);
pdgbinfo_(OUTFILE, &NOUT, &TRANS, &NMAT, NVAL, &ntests, &NBW,
BWLVAL, BWUVAL, &ntests, &NNB, NBVAL, &ntests, &NNR,
NRVAL, &ntests, &NNBR, NBRVAL, &ntests, &NGRIDS, PVAL,
&ntests, QVAL, &ntests, &THRESH, MEM, &IAM, &NPROCS);
CHECK = (THRESH >= 0.0f);
/*! Print headings: */
fprintf(stdout, "\n");
fprintf(stdout, "TIME TR N BWL BWU NB NRHS P Q L*U Time Slv Time MFLOPS MFLOP2 CHECK\n");
fprintf(stdout, "---- -- ------ --- --- ---- ----- ---- ---- -------- -------- -------- -------- ------\n");
fprintf(stdout, "\n");
fflush(stdout);
/*! Loop over different process grids: */
for (I = 1; I <= NGRIDS; I++) {
NPROW = PVAL[I - 1];
NPCOL = QVAL[I - 1];
fprintf(stdout, "Solving for a %d x %d grid!\n", NPROW, NPCOL);
}
free(MEM);
Cblacs_exit(0);
exit(EXIT_SUCCESS);
}
Please excuse the amount of commented-out variables, but like I said, I am basically translating the reference driver, from scratch.
My problem is that it suddenly started seg-faulting, as soon as I wrote the for loop... it was working before. That is a clear sign of a funky memory manipulation issue, which probably arises from issues related to the memory management differences between C and Fortran.
At least, that is my guess.
Does anybody has a clue on what may be going wrong?
Upvotes: 1
Views: 624
Answers (3)
Eduardo J. Sanchez
Reputation: 776
Here the suggested modifications:
SUBROUTINE PDGBINFO( CSUMMRY, NOUT, TRANS, NMAT, NVAL, LDNVAL, NBW,
$ BWLVAL, BWUVAL, LDBWVAL, NNB, NBVAL, LDNBVAL,
$ NNR, NRVAL, LDNRVAL, NNBR, NBRVAL, LDNBRVAL,
$ NGRIDS, PVAL, LDPVAL, QVAL, LDQVAL, THRESH,
$ WORK, IAM, NPROCS ) BIND(C)
*
use iso_c_binding
implicit none
*
* .. Scalar Arguments ..
CHARACTER(kind=c_char), intent(out) :: TRANS
CHARACTER(kind=c_char), dimension(*), intent(OUT) :: CSUMMRY
INTEGER(kind=c_int), intent(in) :: IAM, LDNVAL, LDNBVAL,
$ LDNRVAL, LDNBRVAL, LDPVAL,
$ LDBWVAL, LDQVAL
INTEGER(kind=c_int), intent(inout) :: NPROCS
INTEGER(kind=c_int), intent(out) :: NOUT, NMAT, NBW, NNB,
$ NNR, NNBR, NGRIDS
REAL(KIND=C_FLOAT), intent(out) :: THRESH
* ..
* .. Array Arguments ..
INTEGER(kind=c_int), intent(out) :: NBRVAL( LDNBRVAL ), NBVAL( LDNBVAL ),
$ NRVAL( LDNRVAL ), NVAL( LDNVAL ),
$ BWLVAL( LDBWVAL), BWUVAL( LDBWVAL),
$ PVAL( LDPVAL ), QVAL(LDQVAL), WORK( * )
CHARACTER(KIND=c_char,len=100) :: SUMMRY
INTEGER :: c, sumlen
Yet those yield the following compiling-time error:
[ejspeiro@node01 LIN]$ mpif90 -c -O3 pdgbinfo.f
pdgbinfo.f:5.54:
$ WORK, IAM, NPROCS ) BIND(C)
1
Error: Syntax error in SUBROUTINE statement at (1)
pdgbinfo.f:18.21:
CHARACTER(kind=c_char), intent(out) :: TRANS
1
Error: Parameter 'c_char' at (1) has not been declared or is a variable, which does not reduce to a constant expression
Plus a few more!
Once again:
[ejspeiro@node01 LIN]$ mpif90 -v
mpif90 for the Intel(R) MPI Library 3.2 for Linux
Driving: gfortran -v -I/opt/intel/impi/3.2.0.011/include64/gfortran/4.1.0 -I/opt/intel/impi/3.2.0.011/include64 -L/opt/intel/impi/3.2.0.011/lib64 -Xlinker --enable-new-dtags -Xlinker -rpath -Xlinker /opt/intel/impi/3.2.0.011/lib64 -Xlinker -rpath -Xlinker /opt/intel/mpi-rt/3.2 -lmpi -lmpigf -lmpigi -lrt -lpthread -ldl -lgfortranbegin -lgfortran -lm -shared-libgcc
Using built-in specs.
Target: x86_64-redhat-linux6E
Configured with: ../configure --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-bootstrap --enable-shared --enable-threads=posix --enable-checking=release --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --disable-gnu-unique-object --with-as=/usr/libexec/binutils220/as --enable-languages=c,c++,fortran --disable-libgcj --with-mpfr=/builddir/build/BUILD/gcc-4.4.7-20120601/obj-x86_64-redhat-linux6E/mpfr-install/ --with-ppl=/builddir/build/BUILD/gcc-4.4.7-20120601/obj-x86_64-redhat-linux6E/ppl-install --with-cloog=/builddir/build/BUILD/gcc-4.4.7-20120601/obj-x86_64-redhat-linux6E/cloog-install --with-tune=generic --with-arch_32=i586 --build=x86_64-redhat-linux6E
Thread model: posix
gcc version 4.4.7 20120313 (Red Hat 4.4.7-1) (GCC)
COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/libexec/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/libexec/gcc/x86_64-redhat-linux6E/:/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/lib/gcc/x86_64-redhat-linux6E/:/usr/libexec/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/libexec/gcc/x86_64-redhat-linux6E/:/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/lib/gcc/x86_64-redhat-linux6E/
LIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/:/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/app/intel/mkl/10.1.0.015/lib/em64t/:/opt/intel/Compiler/11.0/083/ipp/em64t/lib/:/opt/intel/Compiler/11.0/083/mkl/lib/em64t/:/opt/intel/Compiler/11.0/083/tbb/em64t/cc4.1.0_libc2.4_kernel2.6.16.21/lib/:/opt/intel/Compiler/11.0/083/ipp/em64t/lib/:/opt/intel/Compiler/11.0/083/mkl/lib/em64t/:/opt/intel/Compiler/11.0/083/tbb/em64t/cc4.1.0_libc2.4_kernel2.6.16.21/lib/:/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../../:/lib/:/usr/lib/
COLLECT_GCC_OPTIONS='-v' '-I/opt/intel/impi/3.2.0.011/include64/gfortran/4.1.0' '-I/opt/intel/impi/3.2.0.011/include64' '-L/opt/intel/impi/3.2.0.011/lib64' '-shared-libgcc' '-mtune=generic'
/usr/libexec/gcc/x86_64-redhat-linux6E/4.4.7/collect2 --eh-frame-hdr -m elf_x86_64 --hash-style=gnu -dynamic-linker /lib64/ld-linux-x86-64.so.2 /usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/crtbegin.o -L/opt/intel/impi/3.2.0.011/lib64 -L/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7 -L/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7 -L/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/app/intel/mkl/10.1.0.015/lib/em64t -L/opt/intel/Compiler/11.0/083/ipp/em64t/lib -L/opt/intel/Compiler/11.0/083/mkl/lib/em64t -L/opt/intel/Compiler/11.0/083/tbb/em64t/cc4.1.0_libc2.4_kernel2.6.16.21/lib -L/opt/intel/Compiler/11.0/083/ipp/em64t/lib -L/opt/intel/Compiler/11.0/083/mkl/lib/em64t -L/opt/intel/Compiler/11.0/083/tbb/em64t/cc4.1.0_libc2.4_kernel2.6.16.21/lib -L/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../.. --enable-new-dtags -rpath /opt/intel/impi/3.2.0.011/lib64 -rpath /opt/intel/mpi-rt/3.2 -lmpi -lmpigf -lmpigi -lrt -lpthread -ldl -lgfortranbegin -lgfortran -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/crtend.o /usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/../../../../lib64/crtn.o
/usr/lib/gcc/x86_64-redhat-linux6E/4.4.7/libgfortranbegin.a(fmain.o): In function `main':
(.text+0x26): undefined reference to `MAIN__'
collect2: ld returned 1 exit status
Upvotes: 0
Jonathan Dursi
Reputation: 50937
Here the fundamental issue is actually in the call to pdgbinfo, which was never really meant to be called from C (it's in TESTING, after all); the issue is the strings, which are in fact passed differently from the numeric arrays -- lengths are always passed (so that you can print them, for instance).
You can mock that up on the C side, but it'll be highly compiler dependent. Your best bet is to either pull the strings out of those routines, or to use ISO_C_BINDING to generate portable C interfaces to the relevant fortran routines. Routines with purely numerical arrays in F77 code shouldn't be an issue.
Updated: If you change the beginning of pdgbinfo.f and the subroutine declaration like so:
SUBROUTINE PDGBINFO(CSUMMRY, NOUT, TRANS, NMAT, NVAL, LDNVAL, NBW,
& BWLVAL, BWUVAL, LDBWVAL, NNB, NBVAL, LDNBVAL,
& NNR, NRVAL, LDNRVAL, NNBR, NBRVAL, LDNBRVAL,
& NGRIDS, PVAL, LDPVAL, QVAL, LDQVAL, THRESH,
& WORK, IAM, NPROCS ) BIND(C)
use iso_c_binding
implicit none
*
*
*
* -- ScaLAPACK routine (version 1.7) --
* University of Tennessee, Knoxville, Oak Ridge National Laboratory,
* and University of California, Berkeley.
* November 15, 1997
*
* .. Scalar Arguments ..
CHARACTER(kind=c_char), intent(OUT) :: TRANS
CHARACTER(kind=c_char), dimension(*), intent(OUT) :: CSUMMRY
INTEGER(kind=c_int), intent(IN) :: IAM, LDNVAL, LDNBVAL,
& LDNRVAL, LDNBRVAL, LDPVAL,
& LDBWVAL, LDQVAL
INTEGER(kind=c_int), intent(INOUT) :: NPROCS
INTEGER(kind=c_int), intent(OUT) :: NOUT, NMAT, NBW, NNB,
& NNR, NNBR, NGRIDS
REAL(kind=c_float), intent(OUT) :: THRESH
INTEGER(kind=c_int), intent(OUT) :: NBRVAL( LDNBRVAL ),
& NBVAL( LDNBVAL ),
& NRVAL( LDNRVAL ), NVAL( LDNVAL ),
& BWLVAL( LDBWVAL),BWUVAL( LDBWVAL),
& PVAL( LDPVAL ), QVAL(LDQVAL), WORK( * )
CHARACTER(kind=c_char,len=100) :: SUMMRY
INTEGER :: c,sumlen
And then change where it reads the file to copy the string into the C-array:
*
* Read name and unit number for summary output file
*
READ( NIN, FMT = * ) SUMMRY
** convert to C string
sumlen = len_trim(SUMMRY)
do c=1,sumlen
csummry(c) = summry(c:c)
enddo
csummry(sumlen+1) = c_null_char
You should then be able to get rid of the underscores after pdgbinfo in your C program and have it work in a way that's portable across compiler suites.
Upvotes: 7
Andreas Grapentin
Reputation: 5796
From what I read from your code, you are never actually checking the value of NGRIDS.
my guess is that it's too large and the array indices go beyond limits.
Upvotes: 1
Related Questions
- MPI_Gather gives seg fault in the most basic code
- Toy example of a segfault in Fortran
- MPI debug, segmentation fault issue
- Segmentation fault error due to MPI_comm_size
- Fortran strange segmentation fault
- causes of seg fault in fortran
- Cause of "Segmentation Fault" in a program that depends on the number of MPI Isend/Irecv used?
- MPI send error with derived data type (Fortran)
- MPI Debugging, Segmentation fault?
- Simple MPI code to test MPI_Type_vector crashes with seg fault - why?