searx-qt/searxqt/models/instances.py

########################################################################
#  Searx-qt - Lightweight desktop application for SearX.
#  Copyright (C) 2020  CYBERDEViL
#
#  This file is part of Searx-qt.
#
#  Searx-qt is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Searx-qt is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <https://www.gnu.org/licenses/>.
#
########################################################################

from urllib.parse import urlparse
import random
from operator import itemgetter

from PyQt5.QtCore import (
    QObject,
    pyqtSignal,
    QAbstractTableModel,
    QVariant,
    Qt
)

from searxqt.core.instances import Instance, Stats2
from searxqt.core.engines import Stats2Engines, EnginesModel

from searxqt.utils.string import boolToStr, listToStr

from searxqt.thread import Thread, ThreadManagerProto

from searxqt.translations import _, timeToString


class InstancesModelTypes:
    NotDefined = 0
    Stats2 = 1
    User = 2


class PersistentEnginesModel(EnginesModel, QObject):
    changed = pyqtSignal()

    def __init__(self, enginesModel=None, parent=None):
        EnginesModel.__init__(self)
        QObject.__init__(self, parent)

        self._currentModel = None

        if enginesModel:
            self.setModel(enginesModel)

    def hasModel(self):
        return False if self._currentModel is None else True

    def setModel(self, enginesModel):
        if self._currentModel:
            self._currentModel.deleteLater()
            self._currentModel.changed.disconnect(self.changed)

        self._currentModel = enginesModel
        self._currentModel.changed.connect(self.changed)
        self._data = self._currentModel.data()

        self.changed.emit()


class UserEnginesModel(EnginesModel, QObject):
    changed = pyqtSignal()

    def __init__(self, handler, parent=None):
        QObject.__init__(self, parent)
        EnginesModel.__init__(self, handler)
        handler.changed.connect(self.changed)


class Stats2EnginesModel(Stats2Engines, QObject):
    changed = pyqtSignal()

    def __init__(self, handler, parent=None):
        """
        @param handler: Object containing engines data.
        @type handler: searxqt.models.instances.Stats2Model
        """
        QObject.__init__(self, parent)
        Stats2Engines.__init__(self, handler)
        handler.changed.connect(self.changed)


class Stats2Model(Stats2, ThreadManagerProto):
    changed = pyqtSignal()

    def __init__(self, requestsHandler, parent=None):
        """
        @param requestsHandler:
        @type requestsHandler: core.requests.RequestsHandler
        """
        Stats2.__init__(self, requestsHandler)
        ThreadManagerProto.__init__(self, parent=parent)

    # ThreadManagerProto override
    def currentJobStr(self):
        if self.hasActiveJobs():
            return _("<b>Updating data from:</b> {0}").format(self.URL)
        return ""

    def setData(self, data):
        Stats2.setData(self, data)
        self.changed.emit()

    def updateInstances(self):
        if self._thread:
            return

        self._thread = Thread(
            Stats2.updateInstances,
            args=[self],
            parent=self
        )

        self._thread.finished.connect(
            self.__updateInstancesThreadFinished
        )

        self.threadStarted.emit()
        self._thread.start()

    def __updateInstancesThreadFinished(self):
        if self._thread.result():
            self.changed.emit()

        self._thread.finished.disconnect(
            self.__updateInstancesThreadFinished
        )

        # Wait before deleting because the `finished` signal is emited
        # from the thread itself, so this method could be called before the
        # thread is actually finished and result in a crash.
        self._thread.wait()

        self._thread.deleteLater()
        self._thread = None

        self.threadFinished.emit()

class InstanceModel(Instance):
    def __init__(self, url, data, parent=None):
        Instance.__init__(self, url, data)


class UserInstanceModel(InstanceModel, QObject):
    def __init__(self, url, data, source='user', parent=None):
        QObject.__init__(self, parent=parent)
        InstanceModel.__init__(self, url, data, source)

    @property
    def lastUpdated(self):
        return self._data.get('lastUpdated', 0)


class InstancesModel(QObject):
    InstanceType = InstanceModel
    Type = InstancesModelTypes.NotDefined

    changed = pyqtSignal()

    def __init__(self, handler=None, parent=None):
        """
        """
        QObject.__init__(self, parent=parent)

        self._instances = {}
        self._modelHandler = handler

        if handler:
            self._instances = handler.instances
            handler.changed.connect(self.changed)

    def __contains__(self, url):
        return bool(url in self._instances)

    def __getitem__(self, url):
        return self.InstanceType(url, self._instances[url])

    def __str__(self): return str([url for url in self._instances])

    def __repr__(self): return str(self)

    def __len__(self): return len(self._instances)

    def data(self):
        return self._instances

    def items(self):
        return [
            (url, self.InstanceType(url, data))
            for url, data in self._instances.items()
        ]

    def keys(self): return self._instances.keys()

    def values(self):
        return [
            self.InstanceType(url, data)
            for url, data in self._instances.items()
        ]

    def copy(self):
        return self._instances.copy()


class PersistentInstancesModel(InstancesModel):
    """ This will either hold a Stats2InstancesModel or a UserInstancesModel
    It can be switched during run-time.

    This ensures that no references are made to the underlying model
    outside of this object.
    """
    typeChanged = pyqtSignal(int)  # InstancesModelTypes

    def __init__(self, instancesModel=None, parent=None):
        InstancesModel.__init__(self, parent=parent)

        self.__currentModel = None
        self._modelHandler = None  # Object that manages the model

        if instancesModel:
            self.setModel(instancesModel)

    def hasModel(self):
        return False if self.__currentModel is None else True

    def hasHandler(self):
        return False if self._modelHandler is None else True

    def handler(self):
        # Do not store references to the returned object!
        return self._modelHandler

    def setModel(self, instancesModel):
        if self.__currentModel:
            self.__currentModel.changed.disconnect(self.changed)
            self.__currentModel.deleteLater()

        self.InstanceType = instancesModel.InstanceType
        self.__currentModel = instancesModel
        self.__currentModel.changed.connect(self.changed)
        self._instances = self.__currentModel.data()
        self._modelHandler = instancesModel._modelHandler

        if self.Type != instancesModel.Type:
            self.Type = instancesModel.Type
            self.typeChanged.emit(self.Type)

        self.changed.emit()


class UserInstancesModel(InstancesModel, QObject):
    InstanceType = UserInstanceModel
    Type = InstancesModelTypes.User

    def __init__(self, handler, parent=None):
        InstancesModel.__init__(self, handler, parent=parent)


class Stats2InstancesModel(InstancesModel):
    Type = InstancesModelTypes.Stats2

    def __init__(self, handler, parent=None):
        """
        @param handler:
        @type handler: Stats2Model
        """
        InstancesModel.__init__(self, handler, parent=parent)


class InstanceModelFilter(QObject):
    changed = pyqtSignal()

    def __init__(self, model, parent=None):
        QObject.__init__(self, parent=parent)
        """
        @type model: searxqt.models.instances.PersistentInstancesModel
        """
        self._model = model
        self._current = model.copy()

        self._filter = {
            'ext': [],
            'skipExt': [],
            'version': [],
            'skipVersion': [],
            'url': [],
            'skipUrl': [],
            'asnPrivacy': True,
            'ipv6': False,
            'engines': []
        }

        self._model.changed.connect(self.apply)

    def loadSettings(self, data):
        defaultAsnPrivacy = bool(self._model.Type != InstancesModelTypes.User)
        self.updateKwargs(
            {
                'ext': data.get('ext', []),
                'skipExt': data.get('skipExt', []),
                'version': data.get('version', []),
                'skipVersion': data.get('skipVersion', []),
                'url': data.get('url', []),
                'skipUrl': data.get('skipUrl', []),
                'asnPrivacy': data.get('asnPrivacy', defaultAsnPrivacy),
                'ipv6': data.get('ipv6', False)
                # 'engines': data.get('engines', [])  # TODO
            }
        )

    def saveSettings(self):
        return self.filter()

    def filter(self): return self._filter

    def parentModel(self): return self._model

    def updateKwargs(self, kwargs, commit=True):
        for key in kwargs:
            self._filter[key] = kwargs[key]
        if commit:
            self.apply()

    def apply(self):
        self._current.clear()
        for url, instance in self._model.items():
            if url not in self._filter['url']:  # Url whitelisted
                # Url blacklist
                if self._filter['skipUrl']:
                    if instance.url in self._filter['skipUrl']:
                        continue

                # Network
                if self._filter['skipExt'] or self._filter['ext']:
                    parsedUrl = urlparse(instance.url)
                    extSplit = parsedUrl.hostname.split('.')
                    ext = extSplit[(len(extSplit)-1)]

                    if self._filter['skipExt']:
                        if (ext in self._filter['skipExt']):
                            continue

                    if self._filter['ext']:
                        if (ext not in self._filter['ext']):
                            continue

                # Version
                if self._filter['skipVersion'] or self._filter['version']:
                    if self._filter['version']:
                        if instance.version not in self._filter['version']:
                            continue

                # ASN privacy
                if self._filter['asnPrivacy']:
                    if instance.network.asnPrivacy != 0:
                        continue

                # IPv6
                if self._filter['ipv6']:
                    if not instance.network.ipv6:
                        continue

                # Engines
                if self._filter['engines']:
                    found = False
                    for engine in self._filter['engines']:
                        for e in instance.engines:
                            if e.name == engine:
                                found = True
                                break

                    if not found:
                        continue

            self._current.update({url: instance})

        self.changed.emit()

    def __contains__(self, url): return bool(url in self._current)

    def __iter__(self): return iter(self._current)

    def __getitem__(self, url): return self._current[url]

    def __str__(self): return str([url for url in self])

    def __repr__(self): return str(self)

    def __len__(self): return len(self._current)

    def items(self): return self._current.items()

    def keys(self): return self._current.keys()

    def values(self): return self._current.values()

    def copy(self): return self._current.copy()


class InstanceSelecterModel(QObject):
    optionsChanged = pyqtSignal()
    instanceChanged = pyqtSignal(str)  # instance url

    def __init__(self, model, parent=None):
        QObject.__init__(self, parent=parent)
        """
        @type model: InstancesModelFilter
        """
        self._model = model

        self._currentInstanceUrl = ''

        self._model.changed.connect(self.__modelChanged)

    def __modelChanged(self):
        """ This can happen after example blacklisting all instances.
        """
        if self.currentUrl and self.currentUrl not in self._model:
            self.currentUrl = ""

    @property
    def currentUrl(self): return self._currentInstanceUrl

    @currentUrl.setter
    def currentUrl(self, url):
        self._currentInstanceUrl = url
        self.instanceChanged.emit(url)

    def loadSettings(self, data):
        self.currentUrl = data.get('currentInstance', '')
        self.instanceChanged.emit(self.currentUrl)

    def saveSettings(self):
        return {
            'currentInstance': self.currentUrl
        }

    def getRandomInstances(self, amount=10):
        """ Returns a list of random instance urls.
        """
        return random.sample(list(self._model.keys()),
                             min(amount, len(self._model.keys())))

    def randomInstance(self):
        if self._model.keys():
            self.currentUrl = random.choice(list(self._model.keys()))
        return self.currentUrl


class EnginesTableModel(QAbstractTableModel):
    """ Model used to display engines with their data in a QTableView and
    for adding/removing engines to/from categories.
    """
    def __init__(self, enginesModel, parent):
        """
        @param enginesModel: Contains data about all engines.
        @type enginesModel: searxqt.models.instances.EnginesModel
        """
        QAbstractTableModel.__init__(self, parent)

        self._model = enginesModel  # contains all engines
        self._userModel = None  # see self.setUserModel method
        self._columns = [
            _('Enabled'),
            _('Name'),
            _('Categories'),
            _('Language support'),
            _('Paging'),
            _('SafeSearch'),
            _('Shortcut'),
            _('Time-range support')
        ]
        self._keyIndex = []
        self._catFilter = ""
        self._sort = (0, None)
        self.__genKeyIndexes()

    def setUserModel(self, model):
        """
        @param model: User category model
        @type model: searxqt.models.search.UserCategoryModel
        """
        self.layoutAboutToBeChanged.emit()
        self._userModel = model
        self.layoutChanged.emit()
        self.reSort()

    def __genKeyIndexes(self):
        self._keyIndex.clear()

        if self._catFilter:
            self._keyIndex = [
                key for key, engine in self._model.items()
                if self._catFilter in engine.categories
            ]
        else:
            self._keyIndex = list(self._model.keys())

    def setCatFilter(self, catKey=""):
        """ Filter engines on category.
        """
        self.layoutAboutToBeChanged.emit()
        self._catFilter = catKey
        self.__genKeyIndexes()
        self.reSort()
        self.layoutChanged.emit()

    def getValueByKey(self, key, columnIndex):
        if columnIndex == 0:
            if self._userModel:
                return boolToStr(bool(key in self._userModel.engines))
            return boolToStr(False)
        elif columnIndex == 1:
            return key
        elif columnIndex == 2:
            return listToStr(self._model[key].categories)
        elif columnIndex == 3:
            return boolToStr(self._model[key].languageSupport)
        elif columnIndex == 4:
            return boolToStr(self._model[key].paging)
        elif columnIndex == 5:
            return boolToStr(self._model[key].safesearch)
        elif columnIndex == 6:
            return self._model[key].shortcut
        elif columnIndex == 7:
            return boolToStr(self._model[key].timeRangeSupport)

    def __sort(self, columnIndex, order=Qt.AscendingOrder):
        unsortedList = [
            [key, self.getValueByKey(key, columnIndex)]
            for key in self._keyIndex
        ]
        reverse = False if order == Qt.AscendingOrder else True

        sortedList = sorted(
                unsortedList,
                key=itemgetter(1),
                reverse=reverse
        )

        self._keyIndex.clear()
        for key, value in sortedList:
            self._keyIndex.append(key)

    def reSort(self):
        if self._sort is not None:
            self.sort(self._sort[0], self._sort[1])

    """ QAbstractTableModel reimplementations below
    """
    def rowCount(self, parent): return len(self._keyIndex)

    def columnCount(self, parent):
        return len(self._columns)

    def headerData(self, col, orientation, role):
        if orientation == Qt.Horizontal and role == Qt.DisplayRole:
            return QVariant(self._columns[col])
        return QVariant()

    def sort(self, columnIndex, order=Qt.AscendingOrder):
        self.layoutAboutToBeChanged.emit()

        self._sort = (columnIndex, order)  # backup current sorting
        self.__sort(columnIndex, order=order)

        self.layoutChanged.emit()

    def setData(self, index, value, role):
        if not index.isValid():
            return False

        if role == Qt.CheckStateRole:
            if self._userModel is not None:
                key = self._keyIndex[index.row()]
                if value:
                    self._userModel.addEngine(key)
                else:
                    self._userModel.removeEngine(key)

                self.reSort()

                return True

        return False

    def data(self, index, role):
        if not index.isValid():
            return QVariant()

        if role == Qt.DisplayRole:
            key = self._keyIndex[index.row()]
            return self.getValueByKey(key, index.column())

        elif index.column() == 0 and role == Qt.CheckStateRole:
            if self._userModel is not None:
                key = self._keyIndex[index.row()]
                if key in self._userModel.engines:
                    return Qt.Checked
            return Qt.Unchecked

        return QVariant()

    def flags(self, index):
        flags = (
            Qt.ItemIsSelectable |
            Qt.ItemIsEnabled |
            Qt.ItemNeverHasChildren
        )
        if index.column() == 0:
            flags = flags | Qt.ItemIsUserCheckable
        return flags


class InstanceTableModel(QAbstractTableModel):
    """ `InstancesModel` -> `QAbstractTableModel` adapter model
    """
    class Column:
        def __init__(self, name, route, type_):
            self._name = name
            self._route = route
            self._type = type_

        @property
        def type(self): return self._type

        @property
        def name(self): return self._name

        @property
        def route(self): return self._route

    def __init__(self, instancesModel, parent):
        """
        @param instancesModel: Resource model
        @type instancesModel: InstancesModel
        """
        QAbstractTableModel.__init__(self, parent)
        self._model = instancesModel
        self._currentModelType = instancesModel.parentModel().Type

        self._keyIndex = []  # [key, key, ..]
        self.__currentSorting = (0, Qt.AscendingOrder)

        self._columns = [
            InstanceTableModel.Column('url', 'url', str),
            InstanceTableModel.Column('version', 'version', str),
            InstanceTableModel.Column('engines', 'engines', list),
            InstanceTableModel.Column('tls.version', 'tls.version', str),
            InstanceTableModel.Column(
                    'tls.cert.version',
                    'tls.certificate.version',
                    int),
            InstanceTableModel.Column(
                    'tls.countryName',
                    'tls.certificate.issuer.countryName',
                    str),
            InstanceTableModel.Column(
                    'tls.commonName',
                    'tls.certificate.issuer.commonName',
                    str),
            InstanceTableModel.Column(
                    'tls.organizationName',
                    'tls.certificate.issuer.organizationName',
                    str),
            InstanceTableModel.Column(
                    'network.asnPrivacy',
                    'network.asnPrivacy',
                    str),
            InstanceTableModel.Column(
                    'network.ipv6',
                    'network.ipv6',
                    bool),
            InstanceTableModel.Column('network.ips', 'network.ips', dict)
        ]

        instancesModel.changed.connect(self.__resourceModelChanged)
        instancesModel.parentModel().typeChanged.connect(
            self.__modelTypeChanged
        )

    def __modelTypeChanged(self, newType):
        previousType = self._currentModelType
        if (previousType != InstancesModelTypes.User and
                newType == InstancesModelTypes.User):
            self._columns.append(
                InstanceTableModel.Column(
                    'lastUpdated',
                    'lastUpdated',
                    int
                )
            )
        elif (previousType == InstancesModelTypes.User and
              newType != InstancesModelTypes.User):
            del self._columns[-1]
        self._currentModelType = newType

    def __genKeyIndexes(self):
        self._keyIndex.clear()
        for key in self._model:
            self._keyIndex.append(key)

    def __resourceModelChanged(self):
        self.sort(*self.__currentSorting)

    def getColumns(self): return self._columns

    def getByIndex(self, index):
        """ Returns a Instance it's URL by index.

        @param index: Index of the instance it's url you like to get.
        @type index: int

        @return: Instance url
        @rtype: str
        """
        return self._keyIndex[index]

    def getByUrl(self, url):
        """ Returns a Instancs it's current index by url

        @param url: Url of the instance you want to get the current
                    index of.
        @type url: str

        @returns: Instance index.
        @rtype: int
        """
        return self._keyIndex.index(url)

    def getPropertyValueByIndex(self, index, route):
        obj = self._model[self.getByIndex(index)]
        return self.getPropertyValue(obj, route)

    def getPropertyValue(self, obj, route):
        """ Returns the `Instance` it's desired property.

        @param obj: instance object
        @type obj: Instance

        @param route: traversel path to value through properties.
        @type route: str
        """
        routes = route.split('.')
        propValue = None
        for propName in routes:
            propValue = getattr(obj, propName)
            obj = propValue

        return propValue

    """ QAbstractTableModel reimplementations below
    """
    def rowCount(self, parent): return len(self._model)

    def columnCount(self, parent): return len(self._columns)

    def sort(self, col, order=Qt.AscendingOrder):
        self.layoutAboutToBeChanged.emit()

        route = self._columns[col].route

        unsortedList = []
        for url, instance in self._model.items():
            value = str(
                        self.getPropertyValue(
                                instance,
                                route
                        )
                    )
            unsortedList.append([url, value])

        reverse = False if order == Qt.AscendingOrder else True

        sortedList = sorted(
                unsortedList,
                key=itemgetter(1),
                reverse=reverse
        )

        self._keyIndex.clear()
        for url, value in sortedList:
            self._keyIndex.append(url)

        self.__currentSorting = (col, order)

        self.layoutChanged.emit()

    def headerData(self, col, orientation, role):
        if orientation == Qt.Horizontal and role == Qt.DisplayRole:
            return QVariant(self._columns[col].name)
        return QVariant()

    def data(self, index, role):
        if not index.isValid():
            return QVariant()

        if role == Qt.DisplayRole:
            value = self.getPropertyValueByIndex(
                        index.row(),
                        self._columns[index.column()].route)

            if index.column() == 2:  # engines
                newStr = ''
                for engine in value:
                    if newStr:
                        newStr += ', {0}'.format(engine.name)
                    else:
                        newStr = engine.name
                return newStr

            elif index.column() == 10:  # ips
                return str(value)

            elif index.column() == 11:  # userInstances lastUpdated
                return timeToString(value)

            return value

        return QVariant()