update_currencies.py 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. #!/usr/bin/env python
  2. import re
  3. import unicodedata
  4. import json
  5. # set path
  6. from sys import path
  7. from os.path import realpath, dirname, join
  8. from searx import searx_dir, settings
  9. from searx.engines.wikidata import send_wikidata_query
  10. # ORDER BY (with all the query fields) is important to keep a deterministic result order
  11. # so multiple invokation of this script doesn't change currencies.json
  12. SARQL_REQUEST = """
  13. SELECT DISTINCT ?iso4217 ?unit ?unicode ?label ?alias WHERE {
  14. ?item wdt:P498 ?iso4217; rdfs:label ?label.
  15. OPTIONAL { ?item skos:altLabel ?alias FILTER (LANG (?alias) = LANG(?label)). }
  16. OPTIONAL { ?item wdt:P5061 ?unit. }
  17. OPTIONAL { ?item wdt:P489 ?symbol.
  18. ?symbol wdt:P487 ?unicode. }
  19. MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
  20. MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
  21. FILTER(LANG(?label) IN (%LANGUAGES_SPARQL%)).
  22. }
  23. ORDER BY ?iso4217 ?unit ?unicode ?label ?alias
  24. """
  25. # ORDER BY (with all the query fields) is important to keep a deterministic result order
  26. # so multiple invokation of this script doesn't change currencies.json
  27. SPARQL_WIKIPEDIA_NAMES_REQUEST = """
  28. SELECT DISTINCT ?iso4217 ?article_name WHERE {
  29. ?item wdt:P498 ?iso4217 .
  30. ?article schema:about ?item ;
  31. schema:name ?article_name ;
  32. schema:isPartOf [ wikibase:wikiGroup "wikipedia" ]
  33. MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
  34. MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
  35. FILTER(LANG(?article_name) IN (%LANGUAGES_SPARQL%)).
  36. }
  37. ORDER BY ?iso4217 ?article_name
  38. """
  39. LANGUAGES = settings['locales'].keys()
  40. LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
  41. def remove_accents(name):
  42. return unicodedata.normalize('NFKD', name).lower()
  43. def remove_extra(name):
  44. for c in ('(', ':'):
  45. if c in name:
  46. name = name.split(c)[0].strip()
  47. return name
  48. def _normalize_name(name):
  49. name = re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))
  50. name = remove_extra(name)
  51. return name
  52. def add_currency_name(db, name, iso4217, normalize_name=True):
  53. db_names = db['names']
  54. if normalize_name:
  55. name = _normalize_name(name)
  56. iso4217_set = db_names.setdefault(name, [])
  57. if iso4217 not in iso4217_set:
  58. iso4217_set.insert(0, iso4217)
  59. def add_currency_label(db, label, iso4217, language):
  60. labels = db['iso4217'].setdefault(iso4217, {})
  61. labels[language] = label
  62. def wikidata_request_result_iterator(request):
  63. result = send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
  64. if result is not None:
  65. for r in result['results']['bindings']:
  66. yield r
  67. def fetch_db():
  68. db = {
  69. 'names': {},
  70. 'iso4217': {},
  71. }
  72. for r in wikidata_request_result_iterator(SPARQL_WIKIPEDIA_NAMES_REQUEST):
  73. iso4217 = r['iso4217']['value']
  74. article_name = r['article_name']['value']
  75. article_lang = r['article_name']['xml:lang']
  76. add_currency_name(db, article_name, iso4217)
  77. add_currency_label(db, article_name, iso4217, article_lang)
  78. for r in wikidata_request_result_iterator(SARQL_REQUEST):
  79. iso4217 = r['iso4217']['value']
  80. if 'label' in r:
  81. label = r['label']['value']
  82. label_lang = r['label']['xml:lang']
  83. add_currency_name(db, label, iso4217)
  84. add_currency_label(db, label, iso4217, label_lang)
  85. if 'alias' in r:
  86. add_currency_name(db, r['alias']['value'], iso4217)
  87. if 'unicode' in r:
  88. add_currency_name(db, r['unicode']['value'], iso4217, normalize_name=False)
  89. if 'unit' in r:
  90. add_currency_name(db, r['unit']['value'], iso4217, normalize_name=False)
  91. # reduce memory usage:
  92. # replace lists with one item by the item.
  93. # see searx.search.processors.online_currency.name_to_iso4217
  94. for name in db['names']:
  95. if len(db['names'][name]) == 1:
  96. db['names'][name] = db['names'][name][0]
  97. return db
  98. def get_filename():
  99. return join(join(searx_dir, "data"), "currencies.json")
  100. def main():
  101. #
  102. db = fetch_db()
  103. # static
  104. add_currency_name(db, "euro", 'EUR')
  105. add_currency_name(db, "euros", 'EUR')
  106. add_currency_name(db, "dollar", 'USD')
  107. add_currency_name(db, "dollars", 'USD')
  108. add_currency_name(db, "peso", 'MXN')
  109. add_currency_name(db, "pesos", 'MXN')
  110. with open(get_filename(), 'w', encoding='utf8') as f:
  111. json.dump(db, f, ensure_ascii=False, indent=4)
  112. if __name__ == '__main__':
  113. main()