123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201 |
- """Google (News)
- For detailed description of the *REST-full* API see: `Query Parameter
- Definitions`_. Not all parameters can be applied:
- - num_ : the number of search results is ignored
- - save_ : is ignored / Google-News results are always *SafeSearch*
- .. _Query Parameter Definitions:
- https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
- .. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
- .. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
- """
- import binascii
- import re
- from urllib.parse import urlencode
- from base64 import b64decode
- from random import random
- from lxml import html
- from searx import logger
- from searx.utils import (
- eval_xpath,
- eval_xpath_list,
- eval_xpath_getindex,
- extract_text,
- )
- from searx.engines.google import (
- supported_languages_url,
- _fetch_supported_languages,
- )
- from searx.engines.google import (
- get_lang_info,
- detect_google_sorry,
- )
- about = {
- "website": 'https://news.google.com',
- "wikidata_id": 'Q12020',
- "official_api_documentation": 'https://developers.google.com/custom-search',
- "use_official_api": False,
- "require_api_key": False,
- "results": 'HTML',
- }
- logger = logger.getChild('google news')
- time_range_dict = {
- 'day': 'when:1d',
- 'week': 'when:7d',
- 'month': 'when:1m',
- 'year': 'when:1y',
- }
- categories = ['news']
- paging = False
- use_locale_domain = True
- time_range_support = True
- safesearch = False
- def request(query, params):
- """Google-News search request"""
- lang_info = get_lang_info(
-
- params, supported_languages, language_aliases, False
- )
-
- lang_info['subdomain'] = 'news.google.com'
- ceid = "%s:%s" % (lang_info['country'], lang_info['language'])
-
- if lang_info['params']['hl'] == 'en':
- lang_info['params']['hl'] = 'en-US'
-
-
- if params['time_range']:
- query += ' ' + time_range_dict[params['time_range']]
- query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
- 'q': query,
- **lang_info['params'],
- 'ie': "utf8",
- 'oe': "utf8",
- 'ucbcb': 1,
- 'gl': lang_info['country'],
- }) + ('&ceid=%s' % ceid)
- logger.debug("query_url --> %s", query_url)
- params['url'] = query_url
- logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
- params['cookies']['CONSENT'] = "PENDING+" + str(random()*100)
- params['headers'].update(lang_info['headers'])
- params['headers']['Accept'] = (
- 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
- )
- return params
- def response(resp):
- """Get response from google's search request"""
- results = []
- detect_google_sorry(resp)
-
- dom = html.fromstring(resp.text)
- for result in eval_xpath_list(dom, '//div[@class="xrnccd"]'):
-
-
-
-
-
-
-
-
- jslog = eval_xpath_getindex(result, './article/a/@jslog', 0)
- url = re.findall('http[^;]*', jslog)
- if url:
- url = url[0]
- else:
-
-
- jslog = jslog.split(";")[1].split(':')[1].strip()
- try:
- padding = (4 -(len(jslog) % 4)) * "="
- jslog = b64decode(jslog + padding)
- except binascii.Error:
-
- continue
-
- url = re.findall('http[^;"]*', str(jslog))[0]
-
- title = extract_text(eval_xpath(result, './article/h3[1]'))
-
- content = extract_text(eval_xpath(result, './article/div[1]'))
-
- pub_date = extract_text(eval_xpath(result, './article/div[2]//time'))
- pub_origin = extract_text(eval_xpath(result, './article/div[2]//a'))
- pub_info = []
- if pub_origin:
- pub_info.append(pub_origin)
- if pub_date:
-
-
- pub_info.append(pub_date)
- pub_info = ', '.join(pub_info)
- if pub_info:
- content = pub_info + ': ' + content
-
-
-
- img_src = extract_text(result.xpath('preceding-sibling::a/figure/img/@src'))
- results.append({
- 'url': url,
- 'title': title,
- 'content': content,
- 'img_src': img_src,
- })
-
- return results
|