webutils.py 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. # -*- coding: utf-8 -*-
  2. import os
  3. import csv
  4. import hashlib
  5. import hmac
  6. import re
  7. import inspect
  8. from io import StringIO
  9. from codecs import getincrementalencoder
  10. from searx import logger
  11. VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')
  12. logger = logger.getChild('webutils')
  13. class UnicodeWriter:
  14. """
  15. A CSV writer which will write rows to CSV file "f",
  16. which is encoded in the given encoding.
  17. """
  18. def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
  19. # Redirect output to a queue
  20. self.queue = StringIO()
  21. self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
  22. self.stream = f
  23. self.encoder = getincrementalencoder(encoding)()
  24. def writerow(self, row):
  25. self.writer.writerow(row)
  26. # Fetch UTF-8 output from the queue ...
  27. data = self.queue.getvalue()
  28. data = data.strip('\x00')
  29. # ... and reencode it into the target encoding
  30. data = self.encoder.encode(data)
  31. # write to the target stream
  32. self.stream.write(data.decode())
  33. # empty queue
  34. self.queue.truncate(0)
  35. def writerows(self, rows):
  36. for row in rows:
  37. self.writerow(row)
  38. def get_resources_directory(searx_directory, subdirectory, resources_directory):
  39. if not resources_directory:
  40. resources_directory = os.path.join(searx_directory, subdirectory)
  41. if not os.path.isdir(resources_directory):
  42. raise Exception(resources_directory + " is not a directory")
  43. return resources_directory
  44. def get_themes(templates_path):
  45. """Returns available themes list."""
  46. themes = os.listdir(templates_path)
  47. if '__common__' in themes:
  48. themes.remove('__common__')
  49. return themes
  50. def get_static_files(static_path):
  51. static_files = set()
  52. static_path_length = len(static_path) + 1
  53. for directory, _, files in os.walk(static_path):
  54. for filename in files:
  55. f = os.path.join(directory[static_path_length:], filename)
  56. static_files.add(f)
  57. return static_files
  58. def get_result_templates(templates_path):
  59. result_templates = set()
  60. templates_path_length = len(templates_path) + 1
  61. for directory, _, files in os.walk(templates_path):
  62. if directory.endswith('result_templates'):
  63. for filename in files:
  64. f = os.path.join(directory[templates_path_length:], filename)
  65. result_templates.add(f)
  66. return result_templates
  67. def new_hmac(secret_key, url):
  68. try:
  69. secret_key_bytes = bytes(secret_key, 'utf-8')
  70. except TypeError as err:
  71. if isinstance(secret_key, bytes):
  72. secret_key_bytes = secret_key
  73. else:
  74. raise err
  75. return hmac.new(secret_key_bytes, url, hashlib.sha256).hexdigest()
  76. def prettify_url(url, max_length=74):
  77. if len(url) > max_length:
  78. chunk_len = int(max_length / 2 + 1)
  79. return '{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])
  80. else:
  81. return url
  82. def highlight_content(content, query):
  83. if not content:
  84. return None
  85. # ignoring html contents
  86. # TODO better html content detection
  87. if content.find('<') != -1:
  88. return content
  89. if content.lower().find(query.lower()) > -1:
  90. query_regex = '({0})'.format(re.escape(query))
  91. content = re.sub(query_regex, '<span class="highlight">\\1</span>',
  92. content, flags=re.I | re.U)
  93. else:
  94. regex_parts = []
  95. for chunk in query.split():
  96. chunk = chunk.replace('"', '')
  97. if len(chunk) == 0:
  98. continue
  99. elif len(chunk) == 1:
  100. regex_parts.append('\\W+{0}\\W+'.format(re.escape(chunk)))
  101. else:
  102. regex_parts.append('{0}'.format(re.escape(chunk)))
  103. query_regex = '({0})'.format('|'.join(regex_parts))
  104. content = re.sub(query_regex, '<span class="highlight">\\1</span>',
  105. content, flags=re.I | re.U)
  106. return content
  107. def is_flask_run_cmdline():
  108. """Check if the application was started using "flask run" command line
  109. Inspect the callstack.
  110. See https://github.com/pallets/flask/blob/master/src/flask/__main__.py
  111. Returns:
  112. bool: True if the application was started using "flask run".
  113. """
  114. frames = inspect.stack()
  115. if len(frames) < 2:
  116. return False
  117. return frames[-2].filename.endswith('flask/cli.py')