OpenCl: addition of large amount of arrays

Question

In the code bellow, each work items generate a array sum_qcos_i.
In order to add them I first make local addition using the local array sum_qcos_tmp.
Then I copy every local array in a global one dimensional matrix sum_qcos_part.
I need to add that matrix columns for my purpose, that's what does each work item before using the result. Here is the code

__kernel __attribute__((vec_type_hint(double4))) void energy_forces( const int atom_number, 
                                         const int nvect,
                                 __global double4 *kvect,__global double *qcos,__global double *qsin,
                                 __global double *cst_ewald ,
                                 __global double4 *positions,
                                 __global double4 *forces_r,               
                                 __global double *sum_qcos_part,__global double *sum_qsin_part)                              
{


  int i = 0 ;
  int gti = 0 , ggi = 0 , lti = 0;

  double kr = (double)0.0 ;
  double ss = (double)0.0 , cc = (double)0.0 ;
  double prod = (double)0.0 ;

  double valqcos = 0. , valqsin = 0. ;  

  double4 zeroes_4 = (double4){ 0.0,0.0,0.0,0.0 };

  double sum_qcos_i[NVECTOR_MAX] ;
  double sum_qsin_i[NVECTOR_MAX] ;    

  #if defined NVECTOR_MAX
  __local double sum_qcos_tmp[NVECTOR_MAX] ;
  __local double sum_qsin_tmp[NVECTOR_MAX] ;         
  #endif

  lti = get_local_id(0);
  ggi = get_group_id(0); 

  for (k=0;k<nvect;k++) {         /*k-vectors*/
 sum_qcos_tmp[k] = .0 ;
 sum_qsin_tmp[k] = .0 ;  
 sum_qcos_i[k] = .0 ;
 sum_qsin_i[k] = .0 ;
            }  

  double fk = (double)0.0 ; 
  double4 fr_i = zeroes_4 ;
  double4 kvec_i = zeroes_4;        

  for (gti = get_global_id(0); gti < atom_number; gti += get_global_size(0))
   {

 pos_i = positions[gti];

 for (k=0;k<nvect;k++) {                           /* sum over k-vectors to compute QCOS and QSIN for Ewald sum*/

   prod = dot((double4)pos_i,(double4)kvect[k]);
   ss = (double)sincos(-prod,&cc);
   valqcos = cc ;
   valqsin = ss ; 
//       valqcos = 1. ;
//       valqsin = 1. ;       
   qcos[gti*NVECTOR_MAX+k] = valqcos ;
   qsin[gti*NVECTOR_MAX+k] = valqsin ;
   sum_qcos_i[k] = valqcos ;                     /*  private variable */
   sum_qsin_i[k] = valqsin ;  

                           }                          /* end sum over k-vectors to compute QCOS and QSIN for Ewald sum*/

}    // end for gti 

  int ii = 0 ;

  for ( ii = 0;ii<get_local_size(0);ii++ )
  { 
if (lti == ii)
  { 
 for (k=0;k<nvect;k++) 
    {                   /* k-vectors */               
       sum_qcos_tmp[k] += sum_qcos_i[k] ;     /* accumulates private data to local variable */
       sum_qsin_tmp[k] += sum_qsin_i[k] ; 
    } 
 barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;    
 }
  }

  if (lti == 0)
   {
     for (k=0;k<nvect;k++) {  
      sum_qcos_part[ggi*NVECTOR_MAX+k] = sum_qcos_tmp[k] ;  /* cp local data to global array */    
      sum_qsin_part[ggi*NVECTOR_MAX+k] = sum_qsin_tmp[k] ;
                           }    
   }

  int iii = 0 ;

  for (gti = get_global_id(0); gti < atom_number; gti += get_global_size(0))
   {

 fr_i = zeroes_4 ;

 barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;

 for (k=0;k<nvect;k++) 
  {   

    sum_qcos_i[k] = .0 ;
    sum_qsin_i[k] = .0 ;

   for (iii=0;iii<get_num_groups(0);iii++)
    {  
      sum_qcos_i[k] += sum_qcos_part[iii*NVECTOR_MAX+k] ;
      sum_qsin_i[k] += sum_qsin_part[iii*NVECTOR_MAX+k] ;     
    }

 } 

 barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;

 for (k=0;k<nvect;k++) 
  {

    fk =  ( sum_qcos_i[k]*qsin[gti*NVECTOR_MAX+k] - sum_qsin_i[k]*qcos[gti*NVECTOR_MAX+k] ) ;               
    fr_i += cst_ewald[k] * fk * kvect[k] ;    

  }

#if defined(SCALAR_KERNELS)
forces_r[gti].x = fr_i.x;
forces_r[gti].y = fr_i.y;
forces_r[gti].z = fr_i.z;     
forces_r[gti].w = .0 ;        
#elif defined(VECTOR_KERNELS)
forces_r[gti] = fr_i;   
#endif

  }    // end for gti  

}

This kernel doesn't work and I can't figure why.
Some hints would be very helpful here.
Thank you.

Answer 1

Actually the problem is not solved.
I define the global work size based on the number of particles so that each work item takes care of one particles.
In my calculations, I dot product each particles coordinates with a certain number of vectors.
My issue now is that the result of those dot products depends on the group size. I put the vectors in a array of double4. The global work size goes from something like 1000 to 10000, whereas the number of vectors I use for the dot product is always around 200. I'm wondering whether there is a requirement on arrays size regarding the global work size.

for (gti = get_global_id(0); gti < atom_number; gti += get_global_size(0))
 {
   pos_i = positions[gti];

   for (k=0;k<nvect;k++) {                           /* sum over k-vectors to compute QCOS and QSIN for Ewald sum*/

 prod = dot((double4)pos_i,(double4)kvect[k]);
 ss = (double)sincos(-prod,&cc);
 valqcos = cc ;
 valqsin = ss ;   
 qcos[gti*NVECTOR_MAX+k] = valqcos ;
 qsin[gti*NVECTOR_MAX+k] = valqsin ;
 sum_qcos_i[k] = valqcos ;                     /*  private variable */
 sum_qsin_i[k] = valqsin ;  

                         }                          /* end sum over k-vectors to compute QCOS and QSIN for Ewald sum*/
 }    // end for gtiforgot 

Any hints here?

Answer 2

Adding a barrier made the trick:

for ( ii = 0;ii<get_local_size(0);ii++ )
{ 
if (lti == ii)
  { 
    barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;      
    for (k=0;k<nvect;k++) 
    {                   /* k-vectors */               
      sum_qcos_tmp[k] += sum_qcos_i[k] ;      /* accumulates private data to local variable */
      sum_qsin_tmp[k] += sum_qsin_i[k] ; 
    } 

  }
}

The final of the the vector forces_r is still wrong abut it's always the same now.