PyQt - How to reimplement QAbstractTableModel sorting?

Question

I am developing an application using PyQt5 (5.7.1) with Python 3.5. I use a QTableView to display a long list of record (more than 10,000). I want to be able to sort and filter this list on several columns at the same time.

I tried using a QAbstractTableModel with a QSortFilterProxyModel, reimplementing QSortFilterProxyModel.filterAcceptsRow() to have multicolumn filtering (see this blog post: http://www.dayofthenewdan.com/2013/02/09/Qt_QSortFilterProxyModel.html). but as this method is called for every row, filtering is very slow when there are a large number of rows.

I thought using Pandas for filtering could improve performance. So I created the following PandasTableModel class, which can indeed perform multicolumn filtering very quickly even with a large number of rows, as well as sorting:

import pandas as pd
from PyQt5 import QtCore, QtWidgets


class PandasTableModel(QtCore.QAbstractTableModel):

    def __init__(self,  parent=None, *args):
        super(PandasTableModel,  self).__init__(parent,  *args)
        self._filters = {}
        self._sortBy = []
        self._sortDirection = []
        self._dfSource = pd.DataFrame()
        self._dfDisplay = pd.DataFrame()

    def rowCount(self,  parent=QtCore.QModelIndex()):
        if parent.isValid():
            return 0
        return self._dfDisplay.shape[0]

    def columnCount(self,  parent=QtCore.QModelIndex()):
        if parent.isValid():
            return 0
        return self._dfDisplay.shape[1]

    def data(self, index, role):
        if index.isValid() and role == QtCore.Qt.DisplayRole:
            return QtCore.QVariant(self._dfDisplay.values[index.row()][index.column()])
        return QtCore.QVariant()

    def headerData(self, col, orientation=QtCore.Qt.Horizontal, role=QtCore.Qt.DisplayRole):
        if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
            return QtCore.QVariant(str(self._dfDisplay.columns[col]))
        return QtCore.QVariant()

    def setupModel(self, header, data):
        self._dfSource = pd.DataFrame(data, columns=header)
        self._sortBy = []
        self._sortDirection = []
        self.setFilters({})

    def setFilters(self, filters):
        self.modelAboutToBeReset.emit()
        self._filters = filters
        self.updateDisplay()
        self.modelReset.emit()

    def sort(self, col, order=QtCore.Qt.AscendingOrder):
        #self.layoutAboutToBeChanged.emit()
        column = self._dfDisplay.columns[col]
        ascending = (order == QtCore.Qt.AscendingOrder)
        if column in self._sortBy:
            i = self._sortBy.index(column)
            self._sortBy.pop(i)
            self._sortDirection.pop(i)
        self._sortBy.insert(0, column)
        self._sortDirection.insert(0, ascending)
        self.updateDisplay()
        #self.layoutChanged.emit()
        self.dataChanged.emit(QtCore.QModelIndex(), QtCore.QModelIndex())

    def updateDisplay(self):

        dfDisplay = self._dfSource.copy()

        # Filtering
        cond = pd.Series(True, index = dfDisplay.index)
        for column, value in self._filters.items():
            cond = cond & \
                (dfDisplay[column].str.lower().str.find(str(value).lower()) >= 0)
        dfDisplay = dfDisplay[cond]

        # Sorting
        if len(self._sortBy) != 0:
            dfDisplay.sort_values(by=self._sortBy,
                                ascending=self._sortDirection,
                                inplace=True)

        # Updating
        self._dfDisplay = dfDisplay

This class replicates the behaviour of a QSortFilterProxyModel, except for one aspect. If an item in the table is selected in the QTableView, sorting the table will not affect the selection (e.g. if the first row is selected before sorting, the first row will still be selected after sorting, not the same one as before.

I think the problem is related to the signals which are emitted. For filtering, I used modelAboutToBeReset() and modelReset(), but these signals cancel selection in the QTableView, so they are not suited for sorting. I read there (How to update QAbstractTableModel and QTableView after sorting the data source?) that layoutAboutToBeChanged() and layoutChanged() should be emitted. However, QTableView doesn't update if I use these signals (I don't understand why actually). When emitting dataChanged() once sorting is completed, QTableView is updated, but with the behaviour described above (selection not updated).

You can test this model using the following example :

class Ui_TableFilteringDialog(object):
    def setupUi(self, TableFilteringDialog):
        TableFilteringDialog.setObjectName("TableFilteringDialog")
        TableFilteringDialog.resize(400, 300)
        self.verticalLayout = QtWidgets.QVBoxLayout(TableFilteringDialog)
        self.verticalLayout.setObjectName("verticalLayout")
        self.tableView = QtWidgets.QTableView(TableFilteringDialog)
        self.tableView.setObjectName("tableView")
        self.tableView.setSortingEnabled(True)
        self.tableView.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
        self.verticalLayout.addWidget(self.tableView)
        self.groupBox = QtWidgets.QGroupBox(TableFilteringDialog)
        self.groupBox.setObjectName("groupBox")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.groupBox)
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.formLayout = QtWidgets.QFormLayout()
        self.formLayout.setObjectName("formLayout")
        self.column1Label = QtWidgets.QLabel(self.groupBox)
        self.column1Label.setObjectName("column1Label")
        self.formLayout.setWidget(0, QtWidgets.QFormLayout.LabelRole, self.column1Label)
        self.column1Field = QtWidgets.QLineEdit(self.groupBox)
        self.column1Field.setObjectName("column1Field")
        self.formLayout.setWidget(0, QtWidgets.QFormLayout.FieldRole, self.column1Field)
        self.column2Label = QtWidgets.QLabel(self.groupBox)
        self.column2Label.setObjectName("column2Label")
        self.formLayout.setWidget(1, QtWidgets.QFormLayout.LabelRole, self.column2Label)
        self.column2Field = QtWidgets.QLineEdit(self.groupBox)
        self.column2Field.setObjectName("column2Field")
        self.formLayout.setWidget(1, QtWidgets.QFormLayout.FieldRole, self.column2Field)
        self.verticalLayout_2.addLayout(self.formLayout)
        self.verticalLayout.addWidget(self.groupBox)

        self.retranslateUi(TableFilteringDialog)
        QtCore.QMetaObject.connectSlotsByName(TableFilteringDialog)

    def retranslateUi(self, TableFilteringDialog):
        _translate = QtCore.QCoreApplication.translate
        TableFilteringDialog.setWindowTitle(_translate("TableFilteringDialog", "Dialog"))
        self.groupBox.setTitle(_translate("TableFilteringDialog", "Filters"))
        self.column1Label.setText(_translate("TableFilteringDialog", "Name"))
        self.column2Label.setText(_translate("TableFilteringDialog", "Occupation"))

class TableFilteringDialog(QtWidgets.QDialog):

    def __init__(self, parent=None):
        super(TableFilteringDialog, self).__init__(parent)

        self.ui = Ui_TableFilteringDialog()
        self.ui.setupUi(self)

        self.tableModel = PandasTableModel()
        header = ['Name', 'Occupation']
        data = [
            ['Abe', 'President'],
            ['Angela', 'Chancelor'],
            ['Donald', 'President'],
            ['François', 'President'],
            ['Jinping', 'President'],
            ['Justin', 'Prime minister'],
            ['Theresa', 'Prime minister'],
            ['Vladimir', 'President'],
            ['Donald', 'Duck']
        ]
        self.tableModel.setupModel(header, data)
        self.ui.tableView.setModel(self.tableModel)

        self.ui.column1Field.textEdited.connect(self.filtersEdited)
        self.ui.column2Field.textEdited.connect(self.filtersEdited)

    def filtersEdited(self):
        filters = {}
        values = [
            self.ui.column1Field.text().lower(),
            self.ui.column2Field.text().lower()
        ]
        for col, value in enumerate(values):
            if value == '':
                continue
            column = self.tableModel.headerData(col, QtCore.Qt.Horizontal, QtCore.Qt.DisplayRole).value()
            filters[column]=value
        self.tableModel.setFilters(filters)



if __name__ == '__main__':

    import sys
    app = QtWidgets.QApplication(sys.argv)

    dialog = TableFilteringDialog()
    dialog.show()

    sys.exit(app.exec_()) 

How can I make the selection follow the selected element when sorting ?

Answer 1

Thanks to ekhumoro, I found a solution. The sort function should store the persistent indexes, create new indexes and change them. Here is the code to do so. It seems sorting a bit slower with a lot of records, but this is acceptable.

def sort(self, col, order=QtCore.Qt.AscendingOrder):

    # Storing persistent indexes
    self.layoutAboutToBeChanged.emit()
    oldIndexList = self.persistentIndexList()
    oldIds = self._dfDisplay.index.copy()

    # Sorting data
    column = self._dfDisplay.columns[col]
    ascending = (order == QtCore.Qt.AscendingOrder)
    if column in self._sortBy:
        i = self._sortBy.index(column)
        self._sortBy.pop(i)
        self._sortDirection.pop(i)
    self._sortBy.insert(0, column)
    self._sortDirection.insert(0, ascending)
    self.updateDisplay()

    # Updating persistent indexes
    newIds = self._dfDisplay.index
    newIndexList = []
    for index in oldIndexList:
        id = oldIds[index.row()]
        newRow = newIds.get_loc(id)
        newIndexList.append(self.index(newRow, index.column(), index.parent()))
    self.changePersistentIndexList(oldIndexList, newIndexList)
    self.layoutChanged.emit()
    self.dataChanged.emit(QtCore.QModelIndex(), QtCore.QModelIndex())

edit: for an unknown reason, emitting dataChanged at the end speeds up sorting considerably. I tried to send a LayoutChangedHint with layoutAboutToBeChanged and layoutChanged (e.g. self.layoutChanged.emit([], QtCore.QAbstractItemModel.VerticalSortHing) ), but I get an error that these signals don't take arguments, which is strange considering the signature of these signals described in Qt5's doc.

Anyways, this code gives me the expected result, so that's already that. Understanding why it works is only a bonus ! ^^ If anyone has an explanation, I'd be interested to know though.