update_external_bangs.py 4.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. #!/usr/bin/env python
  2. """
  3. Update searx/data/external_bangs.json using the duckduckgo bangs.
  4. https://duckduckgo.com/newbang loads
  5. * a javascript which provides the bang version ( https://duckduckgo.com/bv1.js )
  6. * a JSON file which contains the bangs ( https://duckduckgo.com/bang.v260.js for example )
  7. This script loads the javascript, then the bangs.
  8. The javascript URL may change in the future ( for example https://duckduckgo.com/bv2.js ),
  9. but most probably it will requires to update RE_BANG_VERSION
  10. """
  11. # pylint: disable=C0116
  12. import json
  13. import re
  14. from os.path import join
  15. import requests
  16. from searx import searx_dir # pylint: disable=E0401 C0413
  17. # from https://duckduckgo.com/newbang
  18. URL_BV1 = 'https://duckduckgo.com/bv1.js'
  19. RE_BANG_VERSION = re.compile(r'\/bang\.v([0-9]+)\.js')
  20. HTTPS_COLON = 'https:'
  21. HTTP_COLON = 'http:'
  22. def get_bang_url():
  23. response = requests.get(URL_BV1)
  24. response.raise_for_status()
  25. r = RE_BANG_VERSION.findall(response.text)
  26. return f'https://duckduckgo.com/bang.v{r[0]}.js', r[0]
  27. def fetch_ddg_bangs(url):
  28. response = requests.get(url)
  29. response.raise_for_status()
  30. return json.loads(response.content.decode())
  31. def merge_when_no_leaf(node):
  32. """Minimize the number of nodes
  33. A -> B -> C
  34. B is child of A
  35. C is child of B
  36. If there are no C equals to '*', then each C are merged into A
  37. For example:
  38. d -> d -> g -> * (ddg*)
  39. -> i -> g -> * (dig*)
  40. becomes
  41. d -> dg -> *
  42. -> ig -> *
  43. """
  44. restart = False
  45. if not isinstance(node, dict):
  46. return
  47. # create a copy of the keys so node can be modified
  48. keys = list(node.keys())
  49. for key in keys:
  50. if key == '*':
  51. continue
  52. value = node[key]
  53. value_keys = list(value.keys())
  54. if '*' not in value_keys:
  55. for value_key in value_keys:
  56. node[key + value_key] = value[value_key]
  57. merge_when_no_leaf(node[key + value_key])
  58. del node[key]
  59. restart = True
  60. else:
  61. merge_when_no_leaf(value)
  62. if restart:
  63. merge_when_no_leaf(node)
  64. def optimize_leaf(parent, parent_key, node):
  65. if not isinstance(node, dict):
  66. return
  67. if len(node) == 1 and '*' in node and parent is not None:
  68. parent[parent_key] = node['*']
  69. else:
  70. for key, value in node.items():
  71. optimize_leaf(node, key, value)
  72. def parse_ddg_bangs(ddg_bangs):
  73. bang_trie = {}
  74. bang_urls = {}
  75. for bang_definition in ddg_bangs:
  76. # bang_list
  77. bang_url = bang_definition['u']
  78. if '{{{s}}}' not in bang_url:
  79. # ignore invalid bang
  80. continue
  81. bang_url = bang_url.replace('{{{s}}}', chr(2))
  82. # only for the https protocol: "https://example.com" becomes "//example.com"
  83. if bang_url.startswith(HTTPS_COLON + '//'):
  84. bang_url = bang_url[len(HTTPS_COLON):]
  85. #
  86. if bang_url.startswith(HTTP_COLON + '//') and bang_url[len(HTTP_COLON):] in bang_urls:
  87. # if the bang_url uses the http:// protocol, and the same URL exists in https://
  88. # then reuse the https:// bang definition. (written //example.com)
  89. bang_def_output = bang_urls[bang_url[len(HTTP_COLON):]]
  90. else:
  91. # normal use case : new http:// URL or https:// URL (without "https:", see above)
  92. bang_rank = str(bang_definition['r'])
  93. bang_def_output = bang_url + chr(1) + bang_rank
  94. bang_def_output = bang_urls.setdefault(bang_url, bang_def_output)
  95. bang_urls[bang_url] = bang_def_output
  96. # bang name
  97. bang = bang_definition['t']
  98. # bang_trie
  99. t = bang_trie
  100. for bang_letter in bang:
  101. t = t.setdefault(bang_letter, {})
  102. t = t.setdefault('*', bang_def_output)
  103. # optimize the trie
  104. merge_when_no_leaf(bang_trie)
  105. optimize_leaf(None, None, bang_trie)
  106. return bang_trie
  107. def get_bangs_filename():
  108. return join(join(searx_dir, "data"), "external_bangs.json")
  109. if __name__ == '__main__':
  110. bangs_url, bangs_version = get_bang_url()
  111. print(f'fetch bangs from {bangs_url}')
  112. output = {
  113. 'version': bangs_version,
  114. 'trie': parse_ddg_bangs(fetch_ddg_bangs(bangs_url))
  115. }
  116. with open(get_bangs_filename(), 'w') as fp:
  117. json.dump(output, fp, ensure_ascii=False, indent=4)