From 88261e111ca2186f080c4048ab41b4c54cd5cf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Bourrel?= Date: Mon, 5 Aug 2019 16:15:40 +0200 Subject: [PATCH] Fix bing engine results count (#1387) This PR fixes the result count from bing which was throwing an (hidden) error and add a validation to avoid reading more results than avalaible. For example : If there is 100 results from some search and we try to get results from 120 to 130, Bing will send back the results from 0 to 10 and no error. If we compare results count with the first parameter of the request we can avoid this "invalid" results. --- searx/engines/bing.py | 37 +++++++--- tests/unit/engines/test_bing.py | 125 +++++++++++++++++++++++--------- 2 files changed, 119 insertions(+), 43 deletions(-) diff --git a/searx/engines/bing.py b/searx/engines/bing.py index ba22cc6b..1e614867 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -13,11 +13,15 @@ @todo publishedDate """ +import re from lxml import html +from searx import logger, utils from searx.engines.xpath import extract_text from searx.url_utils import urlencode from searx.utils import match_language, gen_useragent +logger = logger.getChild('bing engine') + # engine dependent config categories = ['general'] paging = True @@ -30,9 +34,13 @@ base_url = 'https://www.bing.com/' search_string = 'search?{query}&first={offset}' +def _get_offset_from_pageno(pageno): + return (pageno - 1) * 10 + 1 + + # do search-request def request(query, params): - offset = (params['pageno'] - 1) * 10 + 1 + offset = _get_offset_from_pageno(params.get('pageno', 0)) if params['language'] == 'all': lang = 'EN' @@ -53,15 +61,9 @@ def request(query, params): # get response from search-request def response(resp): results = [] + result_len = 0 dom = html.fromstring(resp.text) - - try: - results.append({'number_of_results': int(dom.xpath('//span[@class="sb_count"]/text()')[0] - .split()[0].replace(',', ''))}) - except: - pass - # parse results for result in dom.xpath('//div[@class="sa_cc"]'): link = result.xpath('.//h3/a')[0] @@ -86,7 +88,24 @@ def response(resp): 'title': title, 'content': content}) - # return results + try: + result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()')) + result_len_container = utils.to_string(result_len_container) + if "-" in result_len_container: + # Remove the part "from-to" for paginated request ... + result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:] + + result_len_container = re.sub('[^0-9]', '', result_len_container) + if len(result_len_container) > 0: + result_len = int(result_len_container) + except Exception as e: + logger.debug('result error :\n%s', e) + pass + + if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: + return [] + + results.append({'number_of_results': result_len}) return results diff --git a/tests/unit/engines/test_bing.py b/tests/unit/engines/test_bing.py index 21191ff2..38703473 100644 --- a/tests/unit/engines/test_bing.py +++ b/tests/unit/engines/test_bing.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from collections import defaultdict import mock from searx.engines import bing @@ -10,7 +11,7 @@ class TestBingEngine(SearxTestCase): bing.supported_languages = ['en', 'fr', 'zh-CHS', 'zh-CHT', 'pt-PT', 'pt-BR'] query = u'test_query' dicto = defaultdict(dict) - dicto['pageno'] = 0 + dicto['pageno'] = 1 dicto['language'] = 'fr-FR' params = bing.request(query.encode('utf-8'), dicto) self.assertTrue('url' in params) @@ -23,70 +24,126 @@ class TestBingEngine(SearxTestCase): self.assertTrue('language' in params['url']) def test_response(self): + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr-FR' self.assertRaises(AttributeError, bing.response, None) self.assertRaises(AttributeError, bing.response, []) self.assertRaises(AttributeError, bing.response, '') self.assertRaises(AttributeError, bing.response, '[]') response = mock.Mock(text='') + response.search_params = dicto self.assertEqual(bing.response(response), []) response = mock.Mock(text='') + response.search_params = dicto self.assertEqual(bing.response(response), []) html = """ -
-
-
-

- - This should be the title -

-
-
this.meta.com - - - - -
-

This should be the content.

+
+
+ 23 900 000 résultats
+
    +
    +
    +
    +

    + + This should be the title +

    +
    +
    this.meta.com + + + + +
    +

    This should be the content.

    +
    +
    +
""" response = mock.Mock(text=html) + response.search_params = dicto results = bing.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[-1]['number_of_results'], 23900000) html = """ -
  • -
    -
    -

    - - This should be the title -

    -
    -
    this.meta.com - - - - -
    -

    This should be the content.

    +
    +
    + 9-18 résultats sur 23 900 000
    -
    • +
      +
    1. +
      +
      +

      + + This should be the title +

      +
      +
      this.meta.com + + + + +
      +

      This should be the content.

      +
      +
      2. +
    +
    """ + dicto['pageno'] = 2 response = mock.Mock(text=html) + response.search_params = dicto results = bing.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[-1]['number_of_results'], 23900000) + + html = """ +
    +
    + 23 900 000 résultats +
    +
      +
    1. +
      +
      +

      + + This should be the title +

      +
      +
      this.meta.com + + + + +
      +

      This should be the content.

      +
      +
      2. +
    +
    + """ + dicto['pageno'] = 33900000 + response = mock.Mock(text=html) + response.search_params = dicto + results = bing.response(response) + self.assertEqual(bing.response(response), []) def test_fetch_supported_languages(self): html = """"""