CUDA Crashes for big data set

Question

My computer crashes (I have to manually reset it) when I run my kernel function in a loop for 600+ times (it would not crash if it were like 50 times or so), and I'm not sure what's causing the crash.

My main is as follows:

int main()
{
    int *seam = new int [image->height];
    int width = image->width;
    int height = image->height;

    int *fMC = (int*)malloc(width*height*sizeof(int*));
    int *fNew = (int*)malloc(width*height*sizeof(int*));

    for(int i=0;i<numOfSeams;i++) 
    {
        seam = cpufindSeamV2(fMC,width,height,1);

        fMC = kernel_shiftSeam(fMC,fNew,seam,width,height,nWidth,1);

        for(int k=0;k<height;k++)
        {
            fMC[(nWidth-1)+width*k] = INT_MAX;
        }
    }

and my kernel is :

int* kernel_shiftSeam(int *MCEnergyMat, int *newE, int *seam, int width, int height, int x, int direction)
{
    //time measurement
    float elapsed_time_ms = 0;
    cudaEvent_t start, stop;      //threads per block

    dim3 threads(16,16);
    //blocks
    dim3 blocks((width+threads.x-1)/threads.x, (height+threads.y-1)/threads.y);

    //MCEnergy and Seam arrays on device
    int *device_MC, *device_new, *device_Seam;

    //MCEnergy and Seam arrays on host
    int *host_MC, *host_new, *host_Seam;


    //total number of bytes in array
    int size = width*height*sizeof(int);
    int seamSize;



    if(direction == 1)
    {
        seamSize = height*sizeof(int);
        host_Seam = (int*)malloc(seamSize);
        for(int i=0;i<height;i++)
            host_Seam[i] = seam[i];
    }
    else
    {
        seamSize = width*sizeof(int);
        host_Seam = (int*)malloc(seamSize);
        for(int i=0;i<width;i++)
            host_Seam[i] = seam[i];
    }

     cudaMallocHost((void**)&host_MC, size );
     cudaMallocHost((void**)&host_new, size );

     host_MC = MCEnergyMat;
     host_new = newE;

    //allocate 1D flat array on device
    cudaMalloc((void**)&device_MC, size);
    cudaMalloc((void**)&device_new, size);
    cudaMalloc((void**)&device_Seam, seamSize);

    //copy host array to device
    cudaMemcpy(device_MC, host_MC, size, cudaMemcpyHostToDevice);
    cudaMemcpy(device_new, host_new, size, cudaMemcpyHostToDevice);
    cudaMemcpy(device_Seam, host_Seam, seamSize, cudaMemcpyHostToDevice);

    //measure start time for cpu calculations
    cudaEventCreate(&start);
    cudaEventCreate(&stop);
    cudaEventRecord(start, 0);



    //perform gpu calculations
    if(direction == 1)
    {
      gpu_shiftSeam<<< blocks,threads >>>(device_MC, device_new, device_Seam, width, height, x);
    }

    //measure end time for cpu calcuations
    cudaEventRecord(stop, 0);
    cudaEventSynchronize(stop);
    cudaEventElapsedTime(&elapsed_time_ms, start, stop );

    execTime += elapsed_time_ms;

    //copy out the results back to host
    cudaMemcpy(newE, device_new, size, cudaMemcpyDeviceToHost);

    //free memory
    free(host_Seam);
    cudaFree(host_MC); cudaFree(host_new); 
    cudaFree(device_MC); cudaFree(device_new); cudaFree(device_Seam);

    //destroy event objects
    cudaEventDestroy(start); cudaEventDestroy(stop);

    return newE;
}

So, my program crashes when I call "kernel_shiftSeam" for many times, I also freed the memory using cudaFree so I don't know whether or not its a memory leak problem. It would be great if someone can point me in the right direction.

Answer 1

Could be heap problems. Try reordering the cudaFree statements in your kernel to be LIFO. Check release notes for any newer CUDA drivers that contain heap/leak fixes. On windows try installing process explorer 15.12 or newer as it shows GPU memory usage - and a leaky heap is easy to spot.