update_languages.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206
  1. #!/usr/bin/env python
  2. # This script generates languages.py from intersecting each engine's supported languages.
  3. #
  4. # Output files: searx/data/engines_languages.json and searx/languages.py
  5. import json
  6. from pathlib import Path
  7. from pprint import pformat
  8. from babel import Locale, UnknownLocaleError
  9. from babel.languages import get_global
  10. from searx import settings, searx_dir
  11. from searx.engines import initialize_engines, engines
  12. # Output files.
  13. engines_languages_file = Path(searx_dir) / 'data' / 'engines_languages.json'
  14. languages_file = Path(searx_dir) / 'languages.py'
  15. # Fetchs supported languages for each engine and writes json file with those.
  16. def fetch_supported_languages():
  17. engines_languages = dict()
  18. names = list(engines)
  19. names.sort()
  20. for engine_name in names:
  21. if hasattr(engines[engine_name], 'fetch_supported_languages'):
  22. engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
  23. print("fetched %s languages from engine %s" % (
  24. len(engines_languages[engine_name]), engine_name))
  25. if type(engines_languages[engine_name]) == list:
  26. engines_languages[engine_name] = sorted(engines_languages[engine_name])
  27. # write json file
  28. with open(engines_languages_file, 'w', encoding='utf-8') as f:
  29. json.dump(engines_languages, f, indent=2, sort_keys=True)
  30. return engines_languages
  31. # Get babel Locale object from lang_code if possible.
  32. def get_locale(lang_code):
  33. try:
  34. locale = Locale.parse(lang_code, sep='-')
  35. return locale
  36. except (UnknownLocaleError, ValueError):
  37. return None
  38. # Join all language lists.
  39. def join_language_lists(engines_languages):
  40. language_list = dict()
  41. for engine_name in engines_languages:
  42. for lang_code in engines_languages[engine_name]:
  43. # apply custom fixes if necessary
  44. if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
  45. lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
  46. if lang_code == alias)
  47. locale = get_locale(lang_code)
  48. # ensure that lang_code uses standard language and country codes
  49. if locale and locale.territory:
  50. lang_code = "{lang}-{country}".format(lang=locale.language, country=locale.territory)
  51. short_code = lang_code.split('-')[0]
  52. # add language without country if not in list
  53. if short_code not in language_list:
  54. if locale:
  55. # get language's data from babel's Locale object
  56. language_name = locale.get_language_name().title()
  57. english_name = locale.english_name.split(' (')[0]
  58. elif short_code in engines_languages['wikipedia']:
  59. # get language's data from wikipedia if not known by babel
  60. language_name = engines_languages['wikipedia'][short_code]['name']
  61. english_name = engines_languages['wikipedia'][short_code]['english_name']
  62. else:
  63. language_name = None
  64. english_name = None
  65. # add language to list
  66. language_list[short_code] = {'name': language_name,
  67. 'english_name': english_name,
  68. 'counter': set(),
  69. 'countries': dict()}
  70. # add language with country if not in list
  71. if lang_code != short_code and lang_code not in language_list[short_code]['countries']:
  72. country_name = ''
  73. if locale:
  74. # get country name from babel's Locale object
  75. country_name = locale.get_territory_name()
  76. language_list[short_code]['countries'][lang_code] = {'country_name': country_name,
  77. 'counter': set()}
  78. # count engine for both language_country combination and language alone
  79. language_list[short_code]['counter'].add(engine_name)
  80. if lang_code != short_code:
  81. language_list[short_code]['countries'][lang_code]['counter'].add(engine_name)
  82. return language_list
  83. # Filter language list so it only includes the most supported languages and countries
  84. def filter_language_list(all_languages):
  85. min_engines_per_lang = 15
  86. min_engines_per_country = 10
  87. main_engines = [engine_name for engine_name in engines.keys()
  88. if 'general' in engines[engine_name].categories and
  89. engines[engine_name].supported_languages and
  90. not engines[engine_name].disabled]
  91. # filter list to include only languages supported by most engines or all default general engines
  92. filtered_languages = {code: lang for code, lang
  93. in all_languages.items()
  94. if (len(lang['counter']) >= min_engines_per_lang or
  95. all(main_engine in lang['counter']
  96. for main_engine in main_engines))}
  97. def _copy_lang_data(lang, country_name=None):
  98. new_dict = dict()
  99. new_dict['name'] = all_languages[lang]['name']
  100. new_dict['english_name'] = all_languages[lang]['english_name']
  101. if country_name:
  102. new_dict['country_name'] = country_name
  103. return new_dict
  104. def _country_count(i):
  105. return len(countries[sorted_countries[i]]['counter'])
  106. # for each language get country codes supported by most engines or at least one country code
  107. filtered_languages_with_countries = dict()
  108. for lang, lang_data in filtered_languages.items():
  109. countries = lang_data['countries']
  110. filtered_countries = dict()
  111. # get language's country codes with enough supported engines
  112. for lang_country, country_data in countries.items():
  113. if len(country_data['counter']) >= min_engines_per_country:
  114. filtered_countries[lang_country] = _copy_lang_data(lang, country_data['country_name'])
  115. # add language without countries too if there's more than one country to choose from
  116. if len(filtered_countries) > 1:
  117. filtered_countries[lang] = _copy_lang_data(lang)
  118. elif len(filtered_countries) == 1:
  119. # if there's only one country per language, it's not necessary to show country name
  120. lang_country = next(iter(filtered_countries))
  121. filtered_countries[lang_country]['country_name'] = None
  122. # if no country has enough engines try to get most likely country code from babel
  123. if not filtered_countries:
  124. lang_country = None
  125. subtags = get_global('likely_subtags').get(lang)
  126. if subtags:
  127. country_code = subtags.split('_')[-1]
  128. if len(country_code) == 2:
  129. lang_country = "{lang}-{country}".format(lang=lang, country=country_code)
  130. if lang_country:
  131. filtered_countries[lang_country] = _copy_lang_data(lang)
  132. else:
  133. filtered_countries[lang] = _copy_lang_data(lang)
  134. filtered_languages_with_countries.update(filtered_countries)
  135. return filtered_languages_with_countries
  136. # Write languages.py.
  137. def write_languages_file(languages):
  138. file_headers = (
  139. "# -*- coding: utf-8 -*-",
  140. "# list of language codes",
  141. "# this file is generated automatically by utils/fetch_languages.py",
  142. "language_codes ="
  143. )
  144. language_codes = tuple([
  145. (
  146. code,
  147. languages[code]['name'].split(' (')[0],
  148. languages[code].get('country_name') or '',
  149. languages[code].get('english_name') or ''
  150. ) for code in sorted(languages)
  151. ])
  152. with open(languages_file, 'w') as new_file:
  153. file_content = "{file_headers} \\\n{language_codes}".format(
  154. file_headers='\n'.join(file_headers),
  155. language_codes=pformat(language_codes, indent=4)
  156. )
  157. new_file.write(file_content)
  158. new_file.close()
  159. if __name__ == "__main__":
  160. initialize_engines(settings['engines'])
  161. engines_languages = fetch_supported_languages()
  162. all_languages = join_language_lists(engines_languages)
  163. filtered_languages = filter_language_list(all_languages)
  164. write_languages_file(filtered_languages)