From 3b1122c5fa1bbac49f6e0eb9ff69efb8dca996ea Mon Sep 17 00:00:00 2001 From: rachmadani haryono Date: Thu, 25 Jul 2019 14:17:45 +0800 Subject: [PATCH] [fix] fix duden engine (#1594) --- searx/engines/duden.py | 19 +++++++++++-------- tests/unit/engines/test_duden.py | 28 +++++++++++++++++----------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 881ff9d9..444f18c1 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -11,7 +11,7 @@ from lxml import html, etree import re from searx.engines.xpath import extract_text -from searx.url_utils import quote +from searx.url_utils import quote, urljoin from searx import logger categories = ['general'] @@ -20,7 +20,7 @@ language_support = False # search-url base_url = 'https://www.duden.de/' -search_url = base_url + 'suchen/dudenonline/{query}?page={offset}' +search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}' def request(query, params): @@ -35,7 +35,11 @@ def request(query, params): ''' offset = (params['pageno'] - 1) - params['url'] = search_url.format(offset=offset, query=quote(query)) + if offset == 0: + search_url_fmt = base_url + 'suchen/dudenonline/{query}' + params['url'] = search_url_fmt.format(query=quote(query)) + else: + params['url'] = search_url.format(offset=offset, query=quote(query)) return params @@ -58,12 +62,11 @@ def response(resp): logger.debug("Couldn't read number of results.") pass - for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'): + for result in dom.xpath('//section[not(contains(@class, "essay"))]'): try: - logger.debug("running for %s" % str(result)) - link = result.xpath('.//h2/a')[0] - url = link.attrib.get('href') - title = result.xpath('string(.//h2/a)') + url = result.xpath('.//h2/a')[0].get('href') + url = urljoin(base_url, url) + title = result.xpath('string(.//h2/a)').strip() content = extract_text(result.xpath('.//p')) # append result results.append({'url': url, diff --git a/tests/unit/engines/test_duden.py b/tests/unit/engines/test_duden.py index d9bbfef8..52fc513d 100644 --- a/tests/unit/engines/test_duden.py +++ b/tests/unit/engines/test_duden.py @@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase): def test_request(self): query = 'Haus' dic = defaultdict(dict) - dic['pageno'] = 1 - params = duden.request(query, dic) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('duden.de' in params['url']) + data = [ + [1, 'https://www.duden.de/suchen/dudenonline/Haus'], + [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1'] + ] + for page_no, exp_res in data: + dic['pageno'] = page_no + params = duden.request(query, dic) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('duden.de' in params['url']) + self.assertEqual(params['url'], exp_res) def test_response(self): resp = mock.Mock(text='') self.assertEqual(duden.response(resp), []) html = """ -
-

This is the title also here

-

This is the content

- Zum vollständigen Artikel +
+ + This is the title also here + +

This is the content

""" - resp = mock.Mock(text=html) results = duden.response(resp) @@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase): # testing result (dictionary entry) r = results[0] - self.assertEqual(r['url'], 'https://this.is.the.url/') + self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus') self.assertEqual(r['title'], 'This is the title also here') self.assertEqual(r['content'], 'This is the content')