utils.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834
  1. # SPDX-License-Identifier: AGPL-3.0-or-later
  2. """Utility functions for the engines
  3. """
  4. from __future__ import annotations
  5. import re
  6. import importlib
  7. import importlib.util
  8. import json
  9. import types
  10. from typing import Optional, Union, Any, Set, List, Dict, MutableMapping, Tuple, Callable
  11. from numbers import Number
  12. from os.path import splitext, join
  13. from random import choice
  14. from html.parser import HTMLParser
  15. from html import escape
  16. from urllib.parse import urljoin, urlparse, parse_qs, urlencode
  17. from markdown_it import MarkdownIt
  18. from lxml import html
  19. from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError
  20. from searx import settings
  21. from searx.data import USER_AGENTS, data_dir
  22. from searx.version import VERSION_TAG
  23. from searx.sxng_locales import sxng_locales
  24. from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
  25. from searx import logger
  26. logger = logger.getChild('utils')
  27. XPathSpecType = Union[str, XPath]
  28. _BLOCKED_TAGS = ('script', 'style')
  29. _ECMA_UNESCAPE4_RE = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE)
  30. _ECMA_UNESCAPE2_RE = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
  31. _JS_QUOTE_KEYS_RE = re.compile(r'([\{\s,])(\w+)(:)')
  32. _JS_VOID_RE = re.compile(r'void\s+[0-9]+|void\s*\([0-9]+\)')
  33. _JS_DECIMAL_RE = re.compile(r":\s*\.")
  34. _XPATH_CACHE: Dict[str, XPath] = {}
  35. _LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {}
  36. _FASTTEXT_MODEL: Optional["fasttext.FastText._FastText"] = None # type: ignore
  37. """fasttext model to predict language of a search term"""
  38. SEARCH_LANGUAGE_CODES = frozenset([searxng_locale[0].split('-')[0] for searxng_locale in sxng_locales])
  39. """Languages supported by most searxng engines (:py:obj:`searx.sxng_locales.sxng_locales`)."""
  40. class _NotSetClass: # pylint: disable=too-few-public-methods
  41. """Internal class for this module, do not create instance of this class.
  42. Replace the None value, allow explicitly pass None as a function argument"""
  43. _NOTSET = _NotSetClass()
  44. def searx_useragent() -> str:
  45. """Return the searx User Agent"""
  46. return 'searx/{searx_version} {suffix}'.format(
  47. searx_version=VERSION_TAG, suffix=settings['outgoing']['useragent_suffix']
  48. ).strip()
  49. def gen_useragent(os_string: Optional[str] = None) -> str:
  50. """Return a random browser User Agent
  51. See searx/data/useragents.json
  52. """
  53. return USER_AGENTS['ua'].format(os=os_string or choice(USER_AGENTS['os']), version=choice(USER_AGENTS['versions']))
  54. class _HTMLTextExtractorException(Exception):
  55. """Internal exception raised when the HTML is invalid"""
  56. class _HTMLTextExtractor(HTMLParser):
  57. """Internal class to extract text from HTML"""
  58. def __init__(self):
  59. HTMLParser.__init__(self)
  60. self.result = []
  61. self.tags = []
  62. def handle_starttag(self, tag, attrs):
  63. self.tags.append(tag)
  64. if tag == 'br':
  65. self.result.append(' ')
  66. def handle_endtag(self, tag):
  67. if not self.tags:
  68. return
  69. if tag != self.tags[-1]:
  70. raise _HTMLTextExtractorException()
  71. self.tags.pop()
  72. def is_valid_tag(self):
  73. return not self.tags or self.tags[-1] not in _BLOCKED_TAGS
  74. def handle_data(self, data):
  75. if not self.is_valid_tag():
  76. return
  77. self.result.append(data)
  78. def handle_charref(self, name):
  79. if not self.is_valid_tag():
  80. return
  81. if name[0] in ('x', 'X'):
  82. codepoint = int(name[1:], 16)
  83. else:
  84. codepoint = int(name)
  85. self.result.append(chr(codepoint))
  86. def handle_entityref(self, name):
  87. if not self.is_valid_tag():
  88. return
  89. # codepoint = htmlentitydefs.name2codepoint[name]
  90. # self.result.append(chr(codepoint))
  91. self.result.append(name)
  92. def get_text(self):
  93. return ''.join(self.result).strip()
  94. def error(self, message):
  95. # error handle is needed in <py3.10
  96. # https://github.com/python/cpython/pull/8562/files
  97. raise AssertionError(message)
  98. def html_to_text(html_str: str) -> str:
  99. """Extract text from a HTML string
  100. Args:
  101. * html_str (str): string HTML
  102. Returns:
  103. * str: extracted text
  104. Examples:
  105. >>> html_to_text('Example <span id="42">#2</span>')
  106. 'Example #2'
  107. >>> html_to_text('<style>.span { color: red; }</style><span>Example</span>')
  108. 'Example'
  109. >>> html_to_text(r'regexp: (?<![a-zA-Z]')
  110. 'regexp: (?<![a-zA-Z]'
  111. """
  112. html_str = html_str.replace('\n', ' ').replace('\r', ' ')
  113. html_str = ' '.join(html_str.split())
  114. s = _HTMLTextExtractor()
  115. try:
  116. s.feed(html_str)
  117. except AssertionError:
  118. s = _HTMLTextExtractor()
  119. s.feed(escape(html_str, quote=True))
  120. except _HTMLTextExtractorException:
  121. logger.debug("HTMLTextExtractor: invalid HTML\n%s", html_str)
  122. return s.get_text()
  123. def markdown_to_text(markdown_str: str) -> str:
  124. """Extract text from a Markdown string
  125. Args:
  126. * markdown_str (str): string Markdown
  127. Returns:
  128. * str: extracted text
  129. Examples:
  130. >>> markdown_to_text('[example](https://example.com)')
  131. 'example'
  132. >>> markdown_to_text('## Headline')
  133. 'Headline'
  134. """
  135. html_str = (
  136. MarkdownIt("commonmark", {"typographer": True}).enable(["replacements", "smartquotes"]).render(markdown_str)
  137. )
  138. return html_to_text(html_str)
  139. def extract_text(xpath_results, allow_none: bool = False) -> Optional[str]:
  140. """Extract text from a lxml result
  141. * if xpath_results is list, extract the text from each result and concat the list
  142. * if xpath_results is a xml element, extract all the text node from it
  143. ( text_content() method from lxml )
  144. * if xpath_results is a string element, then it's already done
  145. """
  146. if isinstance(xpath_results, list):
  147. # it's list of result : concat everything using recursive call
  148. result = ''
  149. for e in xpath_results:
  150. result = result + (extract_text(e) or '')
  151. return result.strip()
  152. if isinstance(xpath_results, ElementBase):
  153. # it's a element
  154. text: str = html.tostring(xpath_results, encoding='unicode', method='text', with_tail=False)
  155. text = text.strip().replace('\n', ' ')
  156. return ' '.join(text.split())
  157. if isinstance(xpath_results, (str, Number, bool)):
  158. return str(xpath_results)
  159. if xpath_results is None and allow_none:
  160. return None
  161. if xpath_results is None and not allow_none:
  162. raise ValueError('extract_text(None, allow_none=False)')
  163. raise ValueError('unsupported type')
  164. def normalize_url(url: str, base_url: str) -> str:
  165. """Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path
  166. Args:
  167. * url (str): Relative URL
  168. * base_url (str): Base URL, it must be an absolute URL.
  169. Example:
  170. >>> normalize_url('https://example.com', 'http://example.com/')
  171. 'https://example.com/'
  172. >>> normalize_url('//example.com', 'http://example.com/')
  173. 'http://example.com/'
  174. >>> normalize_url('//example.com', 'https://example.com/')
  175. 'https://example.com/'
  176. >>> normalize_url('/path?a=1', 'https://example.com')
  177. 'https://example.com/path?a=1'
  178. >>> normalize_url('', 'https://example.com')
  179. 'https://example.com/'
  180. >>> normalize_url('/test', '/path')
  181. raise ValueError
  182. Raises:
  183. * lxml.etree.ParserError
  184. Returns:
  185. * str: normalized URL
  186. """
  187. if url.startswith('//'):
  188. # add http or https to this kind of url //example.com/
  189. parsed_search_url = urlparse(base_url)
  190. url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
  191. elif url.startswith('/'):
  192. # fix relative url to the search engine
  193. url = urljoin(base_url, url)
  194. # fix relative urls that fall through the crack
  195. if '://' not in url:
  196. url = urljoin(base_url, url)
  197. parsed_url = urlparse(url)
  198. # add a / at this end of the url if there is no path
  199. if not parsed_url.netloc:
  200. raise ValueError('Cannot parse url')
  201. if not parsed_url.path:
  202. url += '/'
  203. return url
  204. def extract_url(xpath_results, base_url) -> str:
  205. """Extract and normalize URL from lxml Element
  206. Args:
  207. * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)
  208. * base_url (str): Base URL
  209. Example:
  210. >>> def f(s, search_url):
  211. >>> return searx.utils.extract_url(html.fromstring(s), search_url)
  212. >>> f('<span id="42">https://example.com</span>', 'http://example.com/')
  213. 'https://example.com/'
  214. >>> f('https://example.com', 'http://example.com/')
  215. 'https://example.com/'
  216. >>> f('//example.com', 'http://example.com/')
  217. 'http://example.com/'
  218. >>> f('//example.com', 'https://example.com/')
  219. 'https://example.com/'
  220. >>> f('/path?a=1', 'https://example.com')
  221. 'https://example.com/path?a=1'
  222. >>> f('', 'https://example.com')
  223. raise lxml.etree.ParserError
  224. >>> searx.utils.extract_url([], 'https://example.com')
  225. raise ValueError
  226. Raises:
  227. * ValueError
  228. * lxml.etree.ParserError
  229. Returns:
  230. * str: normalized URL
  231. """
  232. if xpath_results == []:
  233. raise ValueError('Empty url resultset')
  234. url = extract_text(xpath_results)
  235. if url:
  236. return normalize_url(url, base_url)
  237. raise ValueError('URL not found')
  238. def dict_subset(dictionary: MutableMapping, properties: Set[str]) -> Dict:
  239. """Extract a subset of a dict
  240. Examples:
  241. >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C'])
  242. {'A': 'a', 'C': 'c'}
  243. >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])
  244. {'A': 'a'}
  245. """
  246. return {k: dictionary[k] for k in properties if k in dictionary}
  247. def humanize_bytes(size, precision=2):
  248. """Determine the *human readable* value of bytes on 1024 base (1KB=1024B)."""
  249. s = ['B ', 'KB', 'MB', 'GB', 'TB']
  250. x = len(s)
  251. p = 0
  252. while size > 1024 and p < x:
  253. p += 1
  254. size = size / 1024.0
  255. return "%.*f %s" % (precision, size, s[p])
  256. def humanize_number(size, precision=0):
  257. """Determine the *human readable* value of a decimal number."""
  258. s = ['', 'K', 'M', 'B', 'T']
  259. x = len(s)
  260. p = 0
  261. while size > 1000 and p < x:
  262. p += 1
  263. size = size / 1000.0
  264. return "%.*f%s" % (precision, size, s[p])
  265. def convert_str_to_int(number_str: str) -> int:
  266. """Convert number_str to int or 0 if number_str is not a number."""
  267. if number_str.isdigit():
  268. return int(number_str)
  269. return 0
  270. def extr(txt: str, begin: str, end: str, default: str = ""):
  271. """Extract the string between ``begin`` and ``end`` from ``txt``
  272. :param txt: String to search in
  273. :param begin: First string to be searched for
  274. :param end: Second string to be searched for after ``begin``
  275. :param default: Default value if one of ``begin`` or ``end`` is not
  276. found. Defaults to an empty string.
  277. :return: The string between the two search-strings ``begin`` and ``end``.
  278. If at least one of ``begin`` or ``end`` is not found, the value of
  279. ``default`` is returned.
  280. Examples:
  281. >>> extr("abcde", "a", "e")
  282. "bcd"
  283. >>> extr("abcde", "a", "z", deafult="nothing")
  284. "nothing"
  285. """
  286. # From https://github.com/mikf/gallery-dl/blob/master/gallery_dl/text.py#L129
  287. try:
  288. first = txt.index(begin) + len(begin)
  289. return txt[first : txt.index(end, first)]
  290. except ValueError:
  291. return default
  292. def int_or_zero(num: Union[List[str], str]) -> int:
  293. """Convert num to int or 0. num can be either a str or a list.
  294. If num is a list, the first element is converted to int (or return 0 if the list is empty).
  295. If num is a str, see convert_str_to_int
  296. """
  297. if isinstance(num, list):
  298. if len(num) < 1:
  299. return 0
  300. num = num[0]
  301. return convert_str_to_int(num)
  302. def is_valid_lang(lang) -> Optional[Tuple[bool, str, str]]:
  303. """Return language code and name if lang describe a language.
  304. Examples:
  305. >>> is_valid_lang('zz')
  306. None
  307. >>> is_valid_lang('uk')
  308. (True, 'uk', 'ukrainian')
  309. >>> is_valid_lang(b'uk')
  310. (True, 'uk', 'ukrainian')
  311. >>> is_valid_lang('en')
  312. (True, 'en', 'english')
  313. >>> searx.utils.is_valid_lang('Español')
  314. (True, 'es', 'spanish')
  315. >>> searx.utils.is_valid_lang('Spanish')
  316. (True, 'es', 'spanish')
  317. """
  318. if isinstance(lang, bytes):
  319. lang = lang.decode()
  320. is_abbr = len(lang) == 2
  321. lang = lang.lower()
  322. if is_abbr:
  323. for l in sxng_locales:
  324. if l[0][:2] == lang:
  325. return (True, l[0][:2], l[3].lower())
  326. return None
  327. for l in sxng_locales:
  328. if l[1].lower() == lang or l[3].lower() == lang:
  329. return (True, l[0][:2], l[3].lower())
  330. return None
  331. def load_module(filename: str, module_dir: str) -> types.ModuleType:
  332. modname = splitext(filename)[0]
  333. modpath = join(module_dir, filename)
  334. # and https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
  335. spec = importlib.util.spec_from_file_location(modname, modpath)
  336. if not spec:
  337. raise ValueError(f"Error loading '{modpath}' module")
  338. module = importlib.util.module_from_spec(spec)
  339. if not spec.loader:
  340. raise ValueError(f"Error loading '{modpath}' module")
  341. spec.loader.exec_module(module)
  342. return module
  343. def to_string(obj: Any) -> str:
  344. """Convert obj to its string representation."""
  345. if isinstance(obj, str):
  346. return obj
  347. if hasattr(obj, '__str__'):
  348. return str(obj)
  349. return repr(obj)
  350. def ecma_unescape(string: str) -> str:
  351. """Python implementation of the unescape javascript function
  352. https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string
  353. https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape
  354. Examples:
  355. >>> ecma_unescape('%u5409')
  356. '吉'
  357. >>> ecma_unescape('%20')
  358. ' '
  359. >>> ecma_unescape('%F3')
  360. 'ó'
  361. """
  362. # "%u5409" becomes "吉"
  363. string = _ECMA_UNESCAPE4_RE.sub(lambda e: chr(int(e.group(1), 16)), string)
  364. # "%20" becomes " ", "%F3" becomes "ó"
  365. string = _ECMA_UNESCAPE2_RE.sub(lambda e: chr(int(e.group(1), 16)), string)
  366. return string
  367. def remove_pua_from_str(string):
  368. """Removes unicode's "PRIVATE USE CHARACTER"s (PUA_) from a string.
  369. .. _PUA: https://en.wikipedia.org/wiki/Private_Use_Areas
  370. """
  371. pua_ranges = ((0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD))
  372. s = []
  373. for c in string:
  374. i = ord(c)
  375. if any(a <= i <= b for (a, b) in pua_ranges):
  376. continue
  377. s.append(c)
  378. return "".join(s)
  379. def get_string_replaces_function(replaces: Dict[str, str]) -> Callable[[str], str]:
  380. rep = {re.escape(k): v for k, v in replaces.items()}
  381. pattern = re.compile("|".join(rep.keys()))
  382. def func(text):
  383. return pattern.sub(lambda m: rep[re.escape(m.group(0))], text)
  384. return func
  385. def get_engine_from_settings(name: str) -> Dict:
  386. """Return engine configuration from settings.yml of a given engine name"""
  387. if 'engines' not in settings:
  388. return {}
  389. for engine in settings['engines']:
  390. if 'name' not in engine:
  391. continue
  392. if name == engine['name']:
  393. return engine
  394. return {}
  395. def get_xpath(xpath_spec: XPathSpecType) -> XPath:
  396. """Return cached compiled XPath
  397. There is no thread lock.
  398. Worst case scenario, xpath_str is compiled more than one time.
  399. Args:
  400. * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
  401. Returns:
  402. * result (bool, float, list, str): Results.
  403. Raises:
  404. * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
  405. * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
  406. """
  407. if isinstance(xpath_spec, str):
  408. result = _XPATH_CACHE.get(xpath_spec, None)
  409. if result is None:
  410. try:
  411. result = XPath(xpath_spec)
  412. except XPathSyntaxError as e:
  413. raise SearxXPathSyntaxException(xpath_spec, str(e.msg)) from e
  414. _XPATH_CACHE[xpath_spec] = result
  415. return result
  416. if isinstance(xpath_spec, XPath):
  417. return xpath_spec
  418. raise TypeError('xpath_spec must be either a str or a lxml.etree.XPath')
  419. def eval_xpath(element: ElementBase, xpath_spec: XPathSpecType):
  420. """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all.
  421. See https://lxml.de/xpathxslt.html#xpath-return-values
  422. Args:
  423. * element (ElementBase): [description]
  424. * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
  425. Returns:
  426. * result (bool, float, list, str): Results.
  427. Raises:
  428. * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
  429. * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
  430. * SearxEngineXPathException: Raise when the XPath can't be evaluated.
  431. """
  432. xpath = get_xpath(xpath_spec)
  433. try:
  434. return xpath(element)
  435. except XPathError as e:
  436. arg = ' '.join([str(i) for i in e.args])
  437. raise SearxEngineXPathException(xpath_spec, arg) from e
  438. def eval_xpath_list(element: ElementBase, xpath_spec: XPathSpecType, min_len: Optional[int] = None):
  439. """Same as eval_xpath, check if the result is a list
  440. Args:
  441. * element (ElementBase): [description]
  442. * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
  443. * min_len (int, optional): [description]. Defaults to None.
  444. Raises:
  445. * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
  446. * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
  447. * SearxEngineXPathException: raise if the result is not a list
  448. Returns:
  449. * result (bool, float, list, str): Results.
  450. """
  451. result = eval_xpath(element, xpath_spec)
  452. if not isinstance(result, list):
  453. raise SearxEngineXPathException(xpath_spec, 'the result is not a list')
  454. if min_len is not None and min_len > len(result):
  455. raise SearxEngineXPathException(xpath_spec, 'len(xpath_str) < ' + str(min_len))
  456. return result
  457. def eval_xpath_getindex(elements: ElementBase, xpath_spec: XPathSpecType, index: int, default=_NOTSET):
  458. """Call eval_xpath_list then get one element using the index parameter.
  459. If the index does not exist, either raise an exception is default is not set,
  460. other return the default value (can be None).
  461. Args:
  462. * elements (ElementBase): lxml element to apply the xpath.
  463. * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath.
  464. * index (int): index to get
  465. * default (Object, optional): Defaults if index doesn't exist.
  466. Raises:
  467. * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
  468. * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
  469. * SearxEngineXPathException: if the index is not found. Also see eval_xpath.
  470. Returns:
  471. * result (bool, float, list, str): Results.
  472. """
  473. result = eval_xpath_list(elements, xpath_spec)
  474. if -len(result) <= index < len(result):
  475. return result[index]
  476. if default == _NOTSET:
  477. # raise an SearxEngineXPathException instead of IndexError
  478. # to record xpath_spec
  479. raise SearxEngineXPathException(xpath_spec, 'index ' + str(index) + ' not found')
  480. return default
  481. def _get_fasttext_model() -> "fasttext.FastText._FastText": # type: ignore
  482. global _FASTTEXT_MODEL # pylint: disable=global-statement
  483. if _FASTTEXT_MODEL is None:
  484. import fasttext # pylint: disable=import-outside-toplevel
  485. # Monkey patch: prevent fasttext from showing a (useless) warning when loading a model.
  486. fasttext.FastText.eprint = lambda x: None
  487. _FASTTEXT_MODEL = fasttext.load_model(str(data_dir / 'lid.176.ftz'))
  488. return _FASTTEXT_MODEL
  489. def get_embeded_stream_url(url):
  490. """
  491. Converts a standard video URL into its embed format. Supported services include Youtube,
  492. Facebook, Instagram, TikTok, Dailymotion, and Bilibili.
  493. """
  494. parsed_url = urlparse(url)
  495. iframe_src = None
  496. # YouTube
  497. if parsed_url.netloc in ['www.youtube.com', 'youtube.com'] and parsed_url.path == '/watch' and parsed_url.query:
  498. video_id = parse_qs(parsed_url.query).get('v', [])
  499. if video_id:
  500. iframe_src = 'https://www.youtube-nocookie.com/embed/' + video_id[0]
  501. # Facebook
  502. elif parsed_url.netloc in ['www.facebook.com', 'facebook.com']:
  503. encoded_href = urlencode({'href': url})
  504. iframe_src = 'https://www.facebook.com/plugins/video.php?allowfullscreen=true&' + encoded_href
  505. # Instagram
  506. elif parsed_url.netloc in ['www.instagram.com', 'instagram.com'] and parsed_url.path.startswith('/p/'):
  507. if parsed_url.path.endswith('/'):
  508. iframe_src = url + 'embed'
  509. else:
  510. iframe_src = url + '/embed'
  511. # TikTok
  512. elif (
  513. parsed_url.netloc in ['www.tiktok.com', 'tiktok.com']
  514. and parsed_url.path.startswith('/@')
  515. and '/video/' in parsed_url.path
  516. ):
  517. path_parts = parsed_url.path.split('/video/')
  518. video_id = path_parts[1]
  519. iframe_src = 'https://www.tiktok.com/embed/' + video_id
  520. # Dailymotion
  521. elif parsed_url.netloc in ['www.dailymotion.com', 'dailymotion.com'] and parsed_url.path.startswith('/video/'):
  522. path_parts = parsed_url.path.split('/')
  523. if len(path_parts) == 3:
  524. video_id = path_parts[2]
  525. iframe_src = 'https://www.dailymotion.com/embed/video/' + video_id
  526. # Bilibili
  527. elif parsed_url.netloc in ['www.bilibili.com', 'bilibili.com'] and parsed_url.path.startswith('/video/'):
  528. path_parts = parsed_url.path.split('/')
  529. video_id = path_parts[2]
  530. param_key = None
  531. if video_id.startswith('av'):
  532. video_id = video_id[2:]
  533. param_key = 'aid'
  534. elif video_id.startswith('BV'):
  535. param_key = 'bvid'
  536. iframe_src = (
  537. f'https://player.bilibili.com/player.html?{param_key}={video_id}&high_quality=1&autoplay=false&danmaku=0'
  538. )
  539. return iframe_src
  540. def detect_language(text: str, threshold: float = 0.3, only_search_languages: bool = False) -> Optional[str]:
  541. """Detect the language of the ``text`` parameter.
  542. :param str text: The string whose language is to be detected.
  543. :param float threshold: Threshold filters the returned labels by a threshold
  544. on probability. A choice of 0.3 will return labels with at least 0.3
  545. probability.
  546. :param bool only_search_languages: If ``True``, returns only supported
  547. SearXNG search languages. see :py:obj:`searx.languages`
  548. :rtype: str, None
  549. :returns:
  550. The detected language code or ``None``. See below.
  551. :raises ValueError: If ``text`` is not a string.
  552. The language detection is done by using `a fork`_ of the fastText_ library
  553. (`python fasttext`_). fastText_ distributes the `language identification
  554. model`_, for reference:
  555. - `FastText.zip: Compressing text classification models`_
  556. - `Bag of Tricks for Efficient Text Classification`_
  557. The `language identification model`_ support the language codes
  558. (ISO-639-3)::
  559. af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs
  560. bxr ca cbk ce ceb ckb co cs cv cy da de diq dsb dty dv el eml en eo es
  561. et eu fa fi fr frr fy ga gd gl gn gom gu gv he hi hif hr hsb ht hu hy ia
  562. id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv kw ky la lb lez li
  563. lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn nah
  564. nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru
  565. rue sa sah sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl
  566. tr tt tyv ug uk ur uz vec vep vi vls vo wa war wuu xal xmf yi yo yue zh
  567. By using ``only_search_languages=True`` the `language identification model`_
  568. is harmonized with the SearXNG's language (locale) model. General
  569. conditions of SearXNG's locale model are:
  570. a. SearXNG's locale of a query is passed to the
  571. :py:obj:`searx.locales.get_engine_locale` to get a language and/or region
  572. code that is used by an engine.
  573. b. Most of SearXNG's engines do not support all the languages from `language
  574. identification model`_ and there is also a discrepancy in the ISO-639-3
  575. (fasttext) and ISO-639-2 (SearXNG)handling. Further more, in SearXNG the
  576. locales like ``zh-TH`` (``zh-CN``) are mapped to ``zh_Hant``
  577. (``zh_Hans``) while the `language identification model`_ reduce both to
  578. ``zh``.
  579. .. _a fork: https://github.com/searxng/fasttext-predict
  580. .. _fastText: https://fasttext.cc/
  581. .. _python fasttext: https://pypi.org/project/fasttext/
  582. .. _language identification model: https://fasttext.cc/docs/en/language-identification.html
  583. .. _Bag of Tricks for Efficient Text Classification: https://arxiv.org/abs/1607.01759
  584. .. _`FastText.zip: Compressing text classification models`: https://arxiv.org/abs/1612.03651
  585. """
  586. if not isinstance(text, str):
  587. raise ValueError('text must a str')
  588. r = _get_fasttext_model().predict(text.replace('\n', ' '), k=1, threshold=threshold)
  589. if isinstance(r, tuple) and len(r) == 2 and len(r[0]) > 0 and len(r[1]) > 0:
  590. language = r[0][0].split('__label__')[1]
  591. if only_search_languages and language not in SEARCH_LANGUAGE_CODES:
  592. return None
  593. return language
  594. return None
  595. def js_variable_to_python(js_variable):
  596. """Convert a javascript variable into JSON and then load the value
  597. It does not deal with all cases, but it is good enough for now.
  598. chompjs has a better implementation.
  599. """
  600. # when in_string is not None, it contains the character that has opened the string
  601. # either simple quote or double quote
  602. in_string = None
  603. # cut the string:
  604. # r"""{ a:"f\"irst", c:'sec"ond'}"""
  605. # becomes
  606. # ['{ a:', '"', 'f\\', '"', 'irst', '"', ', c:', "'", 'sec', '"', 'ond', "'", '}']
  607. parts = re.split(r'(["\'])', js_variable)
  608. # previous part (to check the escape character antislash)
  609. previous_p = ""
  610. for i, p in enumerate(parts):
  611. # parse characters inside a ECMA string
  612. if in_string:
  613. # we are in a JS string: replace the colon by a temporary character
  614. # so quote_keys_regex doesn't have to deal with colon inside the JS strings
  615. parts[i] = parts[i].replace(':', chr(1))
  616. if in_string == "'":
  617. # the JS string is delimited by simple quote.
  618. # This is not supported by JSON.
  619. # simple quote delimited string are converted to double quote delimited string
  620. # here, inside a JS string, we escape the double quote
  621. parts[i] = parts[i].replace('"', r'\"')
  622. # deal with delimiters and escape character
  623. if not in_string and p in ('"', "'"):
  624. # we are not in string
  625. # but p is double or simple quote
  626. # that's the start of a new string
  627. # replace simple quote by double quote
  628. # (JSON doesn't support simple quote)
  629. parts[i] = '"'
  630. in_string = p
  631. continue
  632. if p == in_string:
  633. # we are in a string and the current part MAY close the string
  634. if len(previous_p) > 0 and previous_p[-1] == '\\':
  635. # there is an antislash just before: the ECMA string continue
  636. continue
  637. # the current p close the string
  638. # replace simple quote by double quote
  639. parts[i] = '"'
  640. in_string = None
  641. if not in_string:
  642. # replace void 0 by null
  643. # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/void
  644. # we are sure there is no string in p
  645. parts[i] = _JS_VOID_RE.sub("null", p)
  646. # update previous_p
  647. previous_p = p
  648. # join the string
  649. s = ''.join(parts)
  650. # add quote around the key
  651. # { a: 12 }
  652. # becomes
  653. # { "a": 12 }
  654. s = _JS_QUOTE_KEYS_RE.sub(r'\1"\2"\3', s)
  655. s = _JS_DECIMAL_RE.sub(":0.", s)
  656. # replace the surogate character by colon
  657. s = s.replace(chr(1), ':')
  658. # load the JSON and return the result
  659. return json.loads(s)