How can I thread this code so late in development?

Question

I have been making a GUI for a genetic algorithm I am working on and I made the mistake of leaving the threading so late simply because I did not (and still don't) know how to do it. So essentially when the start button is clicked the function 'run' starts the whole infinite loop process which actually happens in generation_loop. Each generation the loop checks to see if it should still be running. The idea is that if the stop or pause button has been clicked it will stop looping (with the stop button all the data is cleared with the pause button it remains and the unpause button just sets running to True and calls generation_loop)

So I need to work out a way to make my GUI responsive while generation_loop is running. Here is my code, I tried to minimise it but I am unsure what is important information for threading:

class Window(main_window, QMainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        main_window.__init__(self)
        self.setupUi(self)

        self.scene = QGraphicsScene()
        self.im_view.setScene(self.scene)
        self.setWindowTitle('Fantasy Generator')
        self.running = False
        self.first_run = True
        self.im = Image.new('RGBA', (400, 400), (0, 0, 0, 255))
        self.saved_gens = deque([('A', self.im, self.im, self.im)])
        self.set_save_amount(self.sb_saveamt.value())
        self.population = []

        self.btn_exit.clicked.connect(self.close)
        self.actionQuit.triggered.connect(self.close)
        self.btn_pauser.clicked.connect(self.pause_button)
        self.sb_saveamt.valueChanged[int].connect(self.set_save_amount)
        self.btn_restart.clicked.connect(self.start_button)
        self.btn_loadimage.clicked.connect(self.get_image)
        self.actionLoad_Image.triggered.connect(self.get_image)
        self.gen_sldr.valueChanged[int].connect(self.display_gen)
        self.cb_display.currentIndexChanged.connect(self.change_quality)

        self.has_image = True
        self.display_gen(0)

    def get_image(self):
        pass
        # To save you time I removed the code here. It just sets self.im using a file dialog basically

    def set_save_amount(self, amt):
        if amt == -1:
            self.saved_gens = deque(self.saved_gens)
        else:
            self.saved_gens = deque(self.saved_gens, amt + 1)

    def pause_button(self):
        if self.first_run:
            self.run()
        elif self.running:
            self.running = False
            self.btn_pauser.setText('Resume Execution')
            # pause stuff goes here
        else:
            self.running = True
            self.btn_pauser.setText('Pause Execution')
            self.generation_loop()
            # resume from pause stuff goes here

    def start_button(self):
        if self.first_run:
            self.run()
        else:
            self.end()

# The run function should start the actual process
    def run(self):
        self.btn_restart.setText('End')
        self.btn_pauser.setText('Pause Execution')
        self.first_run = False
        self.running = True

        settings = dict(ind_per_gen=self.sb_ipg.value(), shapes_per_im=self.sb_spi.value(),
                        complexity=self.sb_complexity.value(), mut_rate=self.sb_mutation.value(),
                        cross_chance=self.sb_cross.value(), seed=self.sb_seed.value())
        self.population = Population(self.im, **settings)
        self.generation_loop()

# This is the loop I want to be able to exit out of using buttons
    def generation_loop(self):
        while self.running:
            if self.first_run:
                break
             self.add_generation_data(self.population.next_gen())

    def end(self):
        self.btn_restart.setText('Start')
        self.btn_pauser.setText('Start Execution')
        self.first_run = True
        self.running = False

        self.saved_gens = deque([('A', self.im, self.im, self.im)])
        self.set_save_amount()
        self.display_gen(0)

    def add_generation_data(self, data):
        self.saved_gens.append(data)
        self.gen_sldr.setMaximum(len(self.saved_gens) - 1)
        self.gen_sldr.setValue(len(self.saved_gens) - 1)
        self.display_gen(data[0] + 1)

    def change_quality(self):
        self.display_gen(self.gen_sldr.value())

    def resizeEvent(self, e):
        if self.has_image:
            self.im_view.fitInView(QRectF(0, 0, self.width, self.height), Qt.KeepAspectRatio)
            self.scene.update()

    def display_image(self, image):
        self.scene.clear()
        if image.mode != 'RGBA':
            image = image.convert('RGBA')
        self.width, self.height = image.size
        qim = ImageQt.ImageQt(image)
        pixmap = QPixmap.fromImage(qim)
        self.scene.addPixmap(pixmap)
        self.im_view.fitInView(QRectF(0, 0, self.width, self.height), Qt.KeepAspectRatio)
        self.scene.update()

    def display_gen(self, index):
        self.lcd_cur_gen.display(self.saved_gens[index][0])

        if self.cb_display.currentIndex() == 0:
            self.display_image(self.saved_gens[index][1])
        elif self.cb_display.currentIndex() == 1:
            self.display_image(self.saved_gens[index][2])
        else:
            self.display_image(self.saved_gens[index][3])



if __name__ == '__main__':
    app = QApplication(sys.argv)
    w = Window()
    w.show()
    sys.exit(app.exec_())

EDIT: I also just found at that I can't even change the graphics view from within the generation_loop but it works and changes if I limit the loop

Answer 1

In order to move your long running code to a thread, you need to first identify which parts of the long running code interact with the GUI and which parts don't. The key reason for this is that interacting with the GUI from a secondary thread is forbidden, and will lead to segfaults.

It looks like self.population.next_gen() is the long running bit of the code and doesn't interact with the GUI (although what this does is not provided so I can't be sure) while self.add_generation_data(...) updates the GUI which should be reasonably fast.

As such, this makes it reasonably simple to separate, which I'll show below.

Now, about threads. Python provides threads through the threading module (as the other answers show), however these not are recommended for use with a PyQt application if you want your thread to have any relation to the GUI (see here). PyQt also provides threading via the QThread object, which integrates support for sending and receiving Qt signals (which are thread safe). In short, the QThread has a separate event loop, and processes signals received asynchronously to the main thread, thus leaving the event loop in the main thread to process GUI events (like button clicks).

Typically you create a new class that inherits from QObject, instantiate it and move it to a QThread. Slots (aka methods) in the object that are triggered by a signal emission, then run in the thread.

So you'll want to do something like this

class MyWorker(QObject):
    done = pyqtSignal(object) # you may need to update "object" to the type returned by Population.next_gen()

    def __init__(self, settings):
        # create the population object with whatever settings you need
        # Note that this method runs in the parent thread as you have 
        # yet to move the object to a new thread. It shouldn't cause any
        # problems, but may depend on what the Population class is/does.

        # TODO: I've removed the reference to an image here...
        #it may or may not be thread safe. I can't tell from your code.
        self.population = Population(..., settings)

    @pyqtSlot()
    def next_gen(self):
        new_gen = self.population.next_gen()
        self.done.emit(new_gen)

class Window(....):
    make_next_generation = pyqtSignal() 
    ....

    def run(self):
        self.btn_restart.setText('End')
        self.btn_pauser.setText('Pause Execution')
        self.first_run = False
        self.running = True

        settings = dict(ind_per_gen=self.sb_ipg.value(), shapes_per_im=self.sb_spi.value(),
                        complexity=self.sb_complexity.value(), mut_rate=self.sb_mutation.value(),
                        cross_chance=self.sb_cross.value(), seed=self.sb_seed.value())

        self.setupThread(settings)

    def setupThread(self, settings):
        self.thread = QThread()
        self.worker = MyWorker(settings)    
        self.worker.moveToThread(self.thread)

        # connect a signal in the main thread, to a slot in the worker. 
        # whenever you emit the signal, a new generation will be generated 
        # in the worker thread
        self.make_next_generation.connect(self.worker.next_gen)

        # connect the signal from the worker, to a slot in the main thread.
        # This allows you to update the GUI when a generation has been made
        self.worker.done.connect(self.process_generation)

        # Start thread
        self.thread.start()  

        # emit the signal to start the process!
        self.make_next_generation.emit()

    def process_generation(new_gen):
        # run the GUI side of the code
        # ignore the new generation if the "end" button was clicked
        if not self.first_run:
            self.add_generation_data(new_gen)

        if self.running:
            # make another generation in the thread!
            self.make_next_generation.emit()


    def pause_button(self):
        if self.first_run:
            self.run()
        elif self.running:
            self.running = False
            self.btn_pauser.setText('Resume Execution')
            # pause stuff goes here
        else:
            self.running = True
            self.btn_pauser.setText('Pause Execution')

            # make another generation in the thread!
            self.make_next_generation.emit()

Things to note:

• I haven't included all of your code in my answer. Merge as appropriate.
• I'm unsure what self.im is. It's passed to Population so there might be some thread unsafe behaviour in your code that I can't see. I've left it to you to fix
• I'm familiar with PyQt4, not PyQt5, so there is a possibility some things I've done don't work quite right. It should be easy for you to work out what to change from any error messages that are raised.
• It's a bit messy recreating the thread and worker each time it is started from scratch. You might want to consider moving the instantiation of Population to a method in the worker (one that isn't __init__ and invoking it each time you want to start from scratch (in the same way we trigger a new generation). This would allow you to move pretty much all of setupThread to the Window.__init__ method and then when the start button was clicked, you'd just emit a signal to recreate Population followed by one to make the first generation.

Answer 2

You can use Threading events here.

from threading import Thread, Event

Once you detect the button click,

class MyThread(Thread):
    def __init__(self, the_function, <any input param you want to provide>):
            Thread.__init__(self)
            self.stop_event = Event()
            self.exec_func = the_function

    def set_stop_flag(self, value):

       if value:
           self.stop_event.set()
       else:
           self.stop_event.clear()

    def run(self):
        while True:
            try:
                if not self.stop_event.is_set()
                    self.exec_func()
                else:
                    break # once the event is set, you can break which will kill this thread.
                # To stop busy waiting you can make this thread sleep for some seconds after each cycle.
                import time
                time.sleep(10) # 10 seconds wait before the next cycle.
            except Exception, excep:
                print "Got exception > ", str(excep)

Now in your code you embed this code piece and keep a reference for this thread. Let's say

self.my_thread = MyThread(self.function_to_perform, <blabla>)
self.my_thread.setDaemon(True) # So that you don't have to worry about it when the Main process dies!
self.my_thread.start()

Now once you get a STOP button click event you call

self.my_thread.set_stop_flag(True) # Bingo! Your thread shall quit.