|
@@ -37,6 +37,7 @@ from jsonschema.exceptions import ValidationError, SchemaError
|
|
import random
|
|
import random
|
|
|
|
|
|
from searxqt.core import log
|
|
from searxqt.core import log
|
|
|
|
+from searxqt.core.images import ImagesSettings
|
|
|
|
|
|
HAVE_SOCKS = False
|
|
HAVE_SOCKS = False
|
|
try:
|
|
try:
|
|
@@ -59,7 +60,8 @@ class ErrorType:
|
|
SSLError = 8
|
|
SSLError = 8
|
|
InvalidSchema = 9
|
|
InvalidSchema = 9
|
|
ContentSizeExceeded = 10
|
|
ContentSizeExceeded = 10
|
|
- Other = 11
|
|
|
|
|
|
+ CorruptImage = 11
|
|
|
|
+ Other = 12
|
|
|
|
|
|
|
|
|
|
ErrorTypeStr = {
|
|
ErrorTypeStr = {
|
|
@@ -74,6 +76,7 @@ ErrorTypeStr = {
|
|
ErrorType.SSLError: "SSLError",
|
|
ErrorType.SSLError: "SSLError",
|
|
ErrorType.InvalidSchema: "InvalidSchema",
|
|
ErrorType.InvalidSchema: "InvalidSchema",
|
|
ErrorType.ContentSizeExceeded: "ContentSizeExceeded",
|
|
ErrorType.ContentSizeExceeded: "ContentSizeExceeded",
|
|
|
|
+ ErrorType.CorruptImage: "CorruptImage",
|
|
ErrorType.Other: "Other"
|
|
ErrorType.Other: "Other"
|
|
}
|
|
}
|
|
|
|
|
|
@@ -200,6 +203,26 @@ class JsonResult(Result):
|
|
return json.loads(self._response.content)
|
|
return json.loads(self._response.content)
|
|
|
|
|
|
|
|
|
|
|
|
+if ImagesSettings.supported:
|
|
|
|
+ from io import BytesIO
|
|
|
|
+ from PIL import Image, UnidentifiedImageError
|
|
|
|
+else:
|
|
|
|
+ from searxqt.core.dummy import BytesIO, Image, UnidentifiedImageError
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+class ImageResult(Result):
|
|
|
|
+ def verifyFurther(self):
|
|
|
|
+ content = BytesIO(self.content())
|
|
|
|
+ try:
|
|
|
|
+ Image.open(content, mode='r')
|
|
|
|
+ except UnidentifiedImageError as err:
|
|
|
|
+ self._errType = ErrorType.CorruptImage
|
|
|
|
+ self._err = f"CorruptImage: `{err}` for: {self.url()}"
|
|
|
|
+ except OSError as err:
|
|
|
|
+ self._errType = ErrorType.CorruptImage
|
|
|
|
+ self._err = f"CorruptImage: `{err}` for: {self.url()}"
|
|
|
|
+
|
|
|
|
+
|
|
class ProxyProtocol:
|
|
class ProxyProtocol:
|
|
HTTP = 1
|
|
HTTP = 1
|
|
SOCKS4 = 2
|
|
SOCKS4 = 2
|
|
@@ -248,32 +271,32 @@ class RequestSettings:
|
|
|
|
|
|
def getData(self):
|
|
def getData(self):
|
|
return {
|
|
return {
|
|
- "useragents": self.useragents,
|
|
|
|
- "randomUserAgent": self.randomUserAgent,
|
|
|
|
- "verifySSL": self.verifySSL,
|
|
|
|
- "timeout": self.timeout,
|
|
|
|
- "maxSize": self.maxSize,
|
|
|
|
- "chunkSize": self.chunkSize,
|
|
|
|
- "proxyEnabled": self.proxyEnabled,
|
|
|
|
- "proxyDNS": self.proxyDNS,
|
|
|
|
- "proxyHost": self.proxyHost,
|
|
|
|
- "proxyProtocol": self.proxyProtocol,
|
|
|
|
|
|
+ "useragents": self._useragents,
|
|
|
|
+ "randomUserAgent": self._randomUserAgent,
|
|
|
|
+ "verifySSL": self._verifySSL,
|
|
|
|
+ "timeout": self._timeout,
|
|
|
|
+ "maxSize": self._maxSize,
|
|
|
|
+ "chunkSize": self._chunkSize,
|
|
|
|
+ "proxyEnabled": self._proxyEnabled,
|
|
|
|
+ "proxyDNS": self._proxyDNS,
|
|
|
|
+ "proxyHost": self._proxyHost,
|
|
|
|
+ "proxyProtocol": self._proxyProtocol,
|
|
"extraHeaders": self._extraHeaders
|
|
"extraHeaders": self._extraHeaders
|
|
}
|
|
}
|
|
|
|
|
|
def setData(self, data):
|
|
def setData(self, data):
|
|
- self.useragents.clear()
|
|
|
|
|
|
+ self._useragents.clear()
|
|
for useragent in data.get("useragents", []):
|
|
for useragent in data.get("useragents", []):
|
|
- self.useragents.append(useragent)
|
|
|
|
- self.randomUserAgent = data.get("randomUserAgent", False)
|
|
|
|
- self.verifySSL = data.get("verifySSL", True)
|
|
|
|
- self.timeout = data.get("timeout", 10)
|
|
|
|
- self.maxSize = data.get("maxSize", 10 * 1024 * 1024)
|
|
|
|
- self.chunkSize = data.get("chunkSize", 500 * 1024)
|
|
|
|
- self.proxyEnabled = data.get("proxyEnabled", False)
|
|
|
|
- self.proxyDNS = data.get("proxyDNS", True)
|
|
|
|
- self.proxyHost = data.get("proxyHost", "")
|
|
|
|
- self.proxyProtocol = data.get("proxyProtocol", 0)
|
|
|
|
|
|
+ self._useragents.append(useragent)
|
|
|
|
+ self._randomUserAgent = data.get("randomUserAgent", False)
|
|
|
|
+ self._verifySSL = data.get("verifySSL", True)
|
|
|
|
+ self._timeout = data.get("timeout", 10)
|
|
|
|
+ self._maxSize = data.get("maxSize", 10 * 1024 * 1024)
|
|
|
|
+ self._chunkSize = data.get("chunkSize", 500 * 1024)
|
|
|
|
+ self._proxyEnabled = data.get("proxyEnabled", False)
|
|
|
|
+ self._proxyDNS = data.get("proxyDNS", True)
|
|
|
|
+ self._proxyHost = data.get("proxyHost", "")
|
|
|
|
+ self._proxyProtocol = data.get("proxyProtocol", 0)
|
|
self._extraHeaders = data.get("extraHeaders", {})
|
|
self._extraHeaders = data.get("extraHeaders", {})
|
|
|
|
|
|
self.updateRequestKwargs()
|
|
self.updateRequestKwargs()
|
|
@@ -281,6 +304,10 @@ class RequestSettings:
|
|
""" Settings """
|
|
""" Settings """
|
|
|
|
|
|
@property
|
|
@property
|
|
|
|
+ def headers(self):
|
|
|
|
+ return self._headers
|
|
|
|
+
|
|
|
|
+ @property
|
|
def extraHeaders(self):
|
|
def extraHeaders(self):
|
|
return self._extraHeaders
|
|
return self._extraHeaders
|
|
|
|
|
|
@@ -398,33 +425,118 @@ class RequestSettings:
|
|
kwargs = {
|
|
kwargs = {
|
|
"verify": self.verifySSL,
|
|
"verify": self.verifySSL,
|
|
"timeout": self.timeout,
|
|
"timeout": self.timeout,
|
|
- "headers": self._headers
|
|
|
|
|
|
+ "headers": self.headers
|
|
}
|
|
}
|
|
|
|
|
|
self._headers.clear()
|
|
self._headers.clear()
|
|
self._headers.update(self.extraHeaders)
|
|
self._headers.update(self.extraHeaders)
|
|
|
|
|
|
- if self._proxyEnabled:
|
|
|
|
|
|
+ if self.proxyEnabled:
|
|
kwargs.update({"proxies": self._compileProxies()})
|
|
kwargs.update({"proxies": self._compileProxies()})
|
|
|
|
|
|
self._kwargs.clear()
|
|
self._kwargs.clear()
|
|
self._kwargs.update(kwargs)
|
|
self._kwargs.update(kwargs)
|
|
|
|
|
|
def _getUseragent(self):
|
|
def _getUseragent(self):
|
|
- if not self._useragents:
|
|
|
|
|
|
+ if not self.useragents:
|
|
return ""
|
|
return ""
|
|
|
|
|
|
# Return first useragent string
|
|
# Return first useragent string
|
|
- if len(self._useragents) == 1 or not self._randomUserAgent:
|
|
|
|
- return self._useragents[0]
|
|
|
|
|
|
+ if len(self.useragents) == 1 or not self.randomUserAgent:
|
|
|
|
+ return self.useragents[0]
|
|
|
|
|
|
# Return random useragent
|
|
# Return random useragent
|
|
- return random.choice(self._useragents)
|
|
|
|
|
|
+ return random.choice(self.useragents)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+class RequestSettingsWithParent(RequestSettings):
|
|
|
|
+ # This is read-only when in parent mode
|
|
|
|
+ def __init__(self, parentSettings):
|
|
|
|
+ self._parentSettings = parentSettings
|
|
|
|
+ self._useParent = False
|
|
|
|
+ self._current = self
|
|
|
|
+ RequestSettings.__init__(self)
|
|
|
|
+
|
|
|
|
+ @property
|
|
|
|
+ def useParent(self):
|
|
|
|
+ """! When it returns `True` the settings this object holds will be
|
|
|
|
+ ignored and values from the parent will be returned instead, in
|
|
|
|
+ this case this object should be treated as read-only. When it
|
|
|
|
+ returns `False` this object will behave like a normal
|
|
|
|
+ `RequestSettings` instance.
|
|
|
|
+ """
|
|
|
|
+ return self._useParent
|
|
|
|
+
|
|
|
|
+ @useParent.setter
|
|
|
|
+ def useParent(self, state):
|
|
|
|
+ self._useParent = state
|
|
|
|
+ self._current = self._parentSettings if state else self
|
|
|
|
+
|
|
|
|
+ def getData(self):
|
|
|
|
+ data = RequestSettings.getData(self)
|
|
|
|
+ data.update({"useParent": self.useParent})
|
|
|
|
+ return data
|
|
|
|
+
|
|
|
|
+ def setData(self, data):
|
|
|
|
+ self.useParent = data.get("useParent", False)
|
|
|
|
+ RequestSettings.setData(self, data)
|
|
|
|
+
|
|
|
|
+ """ Override Settings """
|
|
|
|
+
|
|
|
|
+ @RequestSettings.headers.getter
|
|
|
|
+ def headers(self):
|
|
|
|
+ return self._current._headers
|
|
|
|
+
|
|
|
|
+ @RequestSettings.extraHeaders.getter
|
|
|
|
+ def extraHeaders(self):
|
|
|
|
+ return self._current._extraHeaders
|
|
|
|
+
|
|
|
|
+ @RequestSettings.verifySSL.getter
|
|
|
|
+ def verifySSL(self):
|
|
|
|
+ return self._current._verifySSL
|
|
|
|
+
|
|
|
|
+ @RequestSettings.timeout.getter
|
|
|
|
+ def timeout(self):
|
|
|
|
+ return self._current._timeout
|
|
|
|
+
|
|
|
|
+ @RequestSettings.proxyEnabled.getter
|
|
|
|
+ def proxyEnabled(self):
|
|
|
|
+ return self._current._proxyEnabled
|
|
|
|
+
|
|
|
|
+ @RequestSettings.proxyHost.getter
|
|
|
|
+ def proxyHost(self):
|
|
|
|
+ return self._current._proxyHost
|
|
|
|
+
|
|
|
|
+ @RequestSettings.proxyProtocol.getter
|
|
|
|
+ def proxyProtocol(self):
|
|
|
|
+ return self._current._proxyProtocol
|
|
|
|
+
|
|
|
|
+ @RequestSettings.proxyDNS.getter
|
|
|
|
+ def proxyDNS(self):
|
|
|
|
+ return self._current._proxyDNS
|
|
|
|
+
|
|
|
|
+ @RequestSettings.useragents.getter
|
|
|
|
+ def useragents(self):
|
|
|
|
+ return self._current._useragents
|
|
|
|
+
|
|
|
|
+ @RequestSettings.randomUserAgent.getter
|
|
|
|
+ def randomUserAgent(self):
|
|
|
|
+ return self._current._randomUserAgent
|
|
|
|
+
|
|
|
|
|
|
|
|
|
|
class RequestsHandler:
|
|
class RequestsHandler:
|
|
- def __init__(self):
|
|
|
|
- self._settings = RequestSettings()
|
|
|
|
|
|
+ def __init__(self, settings=None):
|
|
|
|
+ """! Handles remote requests.
|
|
|
|
+
|
|
|
|
+ @param settings `RequestSettings` object or `None`. When `None` is
|
|
|
|
+ given it will create a new `RequestSettings` object,
|
|
|
|
+ else the given `settings` object will be used.
|
|
|
|
+ """
|
|
|
|
+ if settings is None:
|
|
|
|
+ self._settings = RequestSettings()
|
|
|
|
+ else:
|
|
|
|
+ self._settings = settings
|
|
|
|
|
|
@property
|
|
@property
|
|
def settings(self):
|
|
def settings(self):
|