Python multiprocessing of a sum

Question

I dont get my code to run, and as others, i have problems in understand how multiprocessing works. here is my code so far

if __name__ == "__main__":
    start = time.clock()
    bins = np.linspace(0,5 * 2 ** 15, 2 ** 15, endpoint=False)  # 1e3
    t_full = np.linspace(0, 0.2, 2 * bins.shape[0], endpoint=False)
    po = Pool()
    res = po.map_async(timeseries, ((m, n, params, bins, 1, t_full, i, i + 1) for i in xrange(2 ** 15)))
    signal = sum(res.get())

where timeseries is given by

def timeseries_para(m, n, params, bins, seed, t, sum_min, sum_max):
    np.random.seed(seed)

    PSD_data = PSD(m, n, params, bins)
    dataReal = np.empty_like(PSD_data)

    for i in range(bins.shape[0]):
        dataReal[i] = np.random.normal(PSD_data[i], 0.1 * PSD_data[i]) 

    plt.loglog(bins, dataReal, 'red')

    dataCOS = np.sqrt(dataReal)
    signal = np.zeros(t.shape[0]) 

    ## Calculating timeseries
    #for i in range(bins.shape[0]):
    for i in range(sum_min, sum_max):
        #start = time.clock()
        signal += dataCOS[i] * np.cos(2 * np.pi * t * bins[i] + random.uniform(0, 2 * np.pi))  
        #print time.clock() - start        

    return signal

My sum goes from 0 up to 2**16, so speeding this up is essential. My problem is, that i first don't know how call my function correct and how i can sum all my replies up.

Thanks for any advice!

Answer 1

This solution works and I am using the vectorized solution proposed here in order to avoid the Python loops:

from multiprocessing import Pool

import numpy as np

def calc(t_full, w, dataCOS):
    thetas = np.multiply.outer((2*np.pi*t_full), w)
    thetas += 2*np.pi*np.random.random(thetas.shape)

    signal = np.cos(thetas)
    signal *= dataCOS

    signal = signal.sum(-1)

    return signal

def parallel_calc(w, dataCOS, t_full, processes, num):
    '''Parallel calculation

    processes : integer
        Number of processes, usually one processor for each process
    num : integer
        Number of sub-divisions for `w` and `dataCOS`
        Must be an exact divisor of `len(w)` and `len(dataCOS)`
    '''
    pool = Pool(processes=processes)
    #
    results = []
    wd = np.vstack((w, dataCOS))
    for wd_s in np.split(wd.T, num):
        w_s = wd_s.T[0]
        d_s = wd_s.T[1]
        results.append(pool.apply_async(calc, (t_full, w_s, d_s)))
    #
    pool.close()
    pool.join()
    return sum((r.get() for r in results))

if __name__ == '__main__':
    w = np.random.random(1000)
    dataCOS = np.random.random(1000)
    t_full = np.arange(2**16)
    #
    parallel_calc(w, dataCOS, t_full, 4, 10)