Python: Multiprocessing Disbehavior

Question

There is a class attribute self.class_variable. Clicking "Press to append One, Two and Three to class_variable list" buttons calls an external to MainWindow() class externalFunc() function used to append 'One','Two','Three'. A single line:

externalFunc(str(self.i))

used to call externalFunc() directly to append a number to the same self.class_variable list.

Pressing 'Press to print class_variable list' prints out the content of the self.class_variable. I have no explanation why

pool.map_async( externalFunc, self.myList )

has no effect on self.class_variable. The values of self.class_variable remain intact. Please advise.

from PyQt4 import QtCore, QtGui
from multiprocessing import Pool

def externalFunc(arg):
    window.setAttrbitute(arg)


class MainWindow(QtGui.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.main_layout = QtGui.QVBoxLayout()
        self.class_variable=['InitialValue']

        self.myList=['One','Two','Three']
        self.i=0

        print_button = QtGui.QPushButton('Press to print class_variable list')      
        print_button.clicked.connect(self.printVariable)
        self.main_layout.addWidget(print_button)

        ok_button = QtGui.QPushButton("Press to append One, Two and Three to class_variable list")
        ok_button.clicked.connect(self.append_from_external_using_multi)
        self.main_layout.addWidget(ok_button)       

        central_widget = QtGui.QWidget()
        central_widget.setLayout(self.main_layout)
        self.setCentralWidget(central_widget)

    def setAttrbitute(self, arg):
        print "Appending arg to class_variable: ", arg
        self.class_variable.append(arg)

    def append_from_external_using_multi(self):
        # calling external function dirctly:
        externalFunc(str(self.i))
        # calling external function via pool.map_async:
        pool = Pool(processes=3)
        pool.map_async( externalFunc, self.myList )
        self.i+=1

    def printVariable(self, arg):
        print "\n\t self.class_variable = ", self.class_variable


if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)
    window = MainWindow()
    window.resize(480, 320)
    window.show()
    sys.exit(app.exec_())

Working code revised based on suggestions posted by J.F. Sebastian

from PyQt4 import QtCore, QtGui
import multiprocessing as mp

def externalFunc(arg):
    window.setAttrbitute(arg)


class MainWindow(QtGui.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.main_layout = QtGui.QVBoxLayout()

        self.class_variable=manager.list(['InitialValue'])

        self.myList=['One','Two','Three']

        self.i=0

        print_button = QtGui.QPushButton('Press to print class_variable list')      
        print_button.clicked.connect(self.printVariable)
        self.main_layout.addWidget(print_button)

        ok_button = QtGui.QPushButton("Press to append One, Two and Three to class_variable list")
        ok_button.clicked.connect(self.append_from_external_using_multi)
        self.main_layout.addWidget(ok_button)    

        central_widget = QtGui.QWidget()
        central_widget.setLayout(self.main_layout)
        self.setCentralWidget(central_widget)

    def setAttrbitute(self, arg):
        print "Appending arg to class_variable: ", arg
        self.class_variable.append(arg)

    def append_from_external_using_multi(self):
        # calling external function dirctly:
        externalFunc(str(self.i))
        # calling external function via pool.map_async:
        pool = mp.Pool(processes=3)

        pool.map_async( externalFunc, self.myList )
        self.i+=1

    def printVariable(self, arg):
        print "\n\t self.class_variable = ", self.class_variable


if __name__ == '__main__':
    manager = mp.Manager()

    app = QtGui.QApplication(sys.argv)
    window = MainWindow()
    window.resize(480, 320)
    window.show()
    sys.exit(app.exec_())

Answer 1

It has no effect because by default different processes do not share objects. In general, you can't share arbitrary Python objects between processes.

Here's a small example that shows the issue:

#!/usr/bin/env python
#XXX broken
import multiprocessing as mp

def f(a):
    L.append(a)

if __name__=="__main__":
   L = ['a']
   pool = mp.Pool(1)
   pool.map(f, 'b')
   print(L) # -> ['a'] it hasn't changed in the parent
   assert 'b' not in L

To fix it, you could use a proxy object via multiprocessing.Manager:

#!/usr/bin/env python
import multiprocessing as mp

def f(a):
    L.append(a)

if __name__=="__main__":
   manager = mp.Manager()
   L = manager.list(['a'])
   pool = mp.Pool(1)
   pool.map(f, 'b')
   print(L)  # -> ['a', 'b'] it has changed in the parent
   assert 'b' in L

Note: you might need to pass L explicitly to the worker process on Windows due to non-fork() semantics e.g.:

def init(ll):
    global L
    L = ll

...
pool = mp.Pool(1, initializer=init, initargs=[L])