From 971179587675e1b47301019130a7874196e3f65f Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 15:36:49 +0200 Subject: [PATCH 1/7] [fix] geektimes.ru xpath fix --- searx/settings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/searx/settings.yml b/searx/settings.yml index 8f57e2e8..26d4f8be 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -321,9 +321,9 @@ engines: engine : xpath paging : True search_url : https://geektimes.ru/search/page{pageno}/?q={query} - url_xpath : //div[@class="search_results"]//a[@class="post__title_link"]/@href - title_xpath : //div[@class="search_results"]//a[@class="post__title_link"] - content_xpath : //div[@class="search_results"]//div[contains(@class, "content")] + url_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"]/@href + title_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"] + content_xpath : //article[contains(@class, "post")]//div[contains(@class, "post__text")] categories : it timeout : 4.0 disabled : True From 2d66df0548bebabfb998098ef43c1b2010887952 Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 15:51:16 +0200 Subject: [PATCH 2/7] [mod]generalfile engine removed --- searx/engines/generalfile.py | 62 ------------------------------------ 1 file changed, 62 deletions(-) delete mode 100644 searx/engines/generalfile.py diff --git a/searx/engines/generalfile.py b/searx/engines/generalfile.py deleted file mode 100644 index 3bb27444..00000000 --- a/searx/engines/generalfile.py +++ /dev/null @@ -1,62 +0,0 @@ -""" - General Files (Files) - - @website http://www.general-files.org - @provide-api no (nothing found) - - @using-api no (because nothing found) - @results HTML (using search portal) - @stable no (HTML can change) - @parse url, title, content - - @todo detect torrents? -""" - -from lxml import html - -# engine dependent config -categories = ['files'] -paging = True - -# search-url -base_url = 'http://www.general-file.com' -search_url = base_url + '/files-{letter}/{query}/{pageno}' - -# specific xpath variables -result_xpath = '//table[@class="block-file"]' -title_xpath = './/h2/a//text()' -url_xpath = './/h2/a/@href' -content_xpath = './/p//text()' - - -# do search-request -def request(query, params): - - params['url'] = search_url.format(query=query, - letter=query[0], - pageno=params['pageno']) - - return params - - -# get response from search-request -def response(resp): - results = [] - - dom = html.fromstring(resp.text) - - # parse results - for result in dom.xpath(result_xpath): - url = result.xpath(url_xpath)[0] - - # skip fast download links - if not url.startswith('/'): - continue - - # append result - results.append({'url': base_url + url, - 'title': ''.join(result.xpath(title_xpath)), - 'content': ''.join(result.xpath(content_xpath))}) - - # return results - return results From 8849606e5f75e6769399aadf06106aee44d0278f Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 16:11:04 +0200 Subject: [PATCH 3/7] [fix] gigablast api fix --- searx/engines/gigablast.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 37933c69..6681de99 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -10,6 +10,7 @@ @parse url, title, content """ +import random from json import loads from time import time from lxml.html import fromstring @@ -32,7 +33,8 @@ search_string = 'search?{query}'\ '&qh=0'\ '&qlang={lang}'\ '&ff={safesearch}'\ - '&rxikd={rxikd}' # random number - 9 digits + '&rxieu={rxieu}'\ + '&rand={rxikd}' # current unix timestamp # specific xpath variables results_xpath = '//response//result' @@ -59,10 +61,12 @@ def request(query, params): else: safesearch = 0 + # rxieu is some kind of hash from the search query, but accepts random atm search_path = search_string.format(query=urlencode({'q': query}), offset=offset, number_of_results=number_of_results, - rxikd=str(time())[:9], + rxikd=int(time() * 1000), + rxieu=random.randint(1000000000,9999999999), lang=language, safesearch=safesearch) From 3182ba7069b2aa2fba1a1b7879bd5f234a4d2868 Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 17:48:07 +0200 Subject: [PATCH 4/7] [fix] google news dom xpath fix --- searx/engines/google_news.py | 4 +- tests/unit/engines/test_google_news.py | 62 +++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index 7344b528..8881d0da 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -67,8 +67,8 @@ def response(resp): for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'): try: r = { - 'url': result.xpath('.//div[@class="_cnc"]//a/@href')[0], - 'title': ''.join(result.xpath('.//div[@class="_cnc"]//h3//text()')), + 'url': result.xpath('.//a[@class="l _PMs"]')[0].attrib.get("href"), + 'title': ''.join(result.xpath('.//a[@class="l _PMs"]//text()')), 'content': ''.join(result.xpath('.//div[@class="st"]//text()')), } except: diff --git a/tests/unit/engines/test_google_news.py b/tests/unit/engines/test_google_news.py index 6454dde4..20a75af4 100644 --- a/tests/unit/engines/test_google_news.py +++ b/tests/unit/engines/test_google_news.py @@ -37,14 +37,64 @@ class TestGoogleNewsEngine(SearxTestCase): self.assertEqual(google_news.response(response), []) html = u""" -
-
A(z) south témájának képe a következőből: CBC.ca

Meet Thuli Madonsela — South Africa's conscience

CBC.ca-9 órával ezelőtt
South African Public Protector
Who really governs South Africa?
Vélemény-News24-2016. dec. 8.
+

Search Results

+
+
+
+ +
+
+
+

+ Example title +

+
+ + Mac & i + + - + + Mar 21, 2016 +
+
Example description
+
+
+
+
+
+ + Story image for searx from Golem.de + +
+

+ Example title 2 +

+
+ + Golem.de + + - + + Oct 4, 2016 +
+
Example description 2
+
+
+
+
+
+ + """ # noqa response = mock.Mock(text=html) results = google_news.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], u'Meet Thuli Madonsela \u2014 South Africa\'s conscience') - self.assertEqual(results[0]['url'], 'http://this.is.the.url') - self.assertEqual(results[0]['content'], 'South African Public Protector') + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['title'], u'Example title') + self.assertEqual(results[0]['url'], 'https://example.com/') + self.assertEqual(results[0]['content'], 'Example description') + self.assertEqual(results[1]['title'], u'Example title 2') + self.assertEqual(results[1]['url'], 'https://example2.com/') + self.assertEqual(results[1]['content'], 'Example description 2') + self.assertEqual(results[1]['img_src'], 'https://example2.com/image.jpg') From bf1ca2ccd37dab4d43386e1c9863043c9dd817f5 Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 18:26:46 +0200 Subject: [PATCH 5/7] [fix] habrahabr.ru xpath fix --- searx/settings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/searx/settings.yml b/searx/settings.yml index 26d4f8be..77d1301b 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -333,9 +333,9 @@ engines: engine : xpath paging : True search_url : https://habrahabr.ru/search/page{pageno}/?q={query} - url_xpath : //div[@class="search_results"]//a[contains(@class, "post__title_link")]/@href - title_xpath : //div[@class="search_results"]//a[contains(@class, "post__title_link")] - content_xpath : //div[@class="search_results"]//div[contains(@class, "content")] + url_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"]/@href + title_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"] + content_xpath : //article[contains(@class, "post")]//div[contains(@class, "post__text")] categories : it timeout : 4.0 disabled : True From b91772eff155f79fdf01be5695f3f15ea24ad17e Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 21:49:19 +0200 Subject: [PATCH 6/7] [fix] gigablast pep8 fix --- searx/engines/gigablast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 6681de99..2bdc97fd 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -66,7 +66,7 @@ def request(query, params): offset=offset, number_of_results=number_of_results, rxikd=int(time() * 1000), - rxieu=random.randint(1000000000,9999999999), + rxieu=random.randint(1000000000, 9999999999), lang=language, safesearch=safesearch) From 9b6307a36518179e7b89d6b1966a1384c52bab5f Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 17:18:04 +0200 Subject: [PATCH 7/7] [fix] gitlab search fixed for proper api usage --- searx/settings.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/searx/settings.yml b/searx/settings.yml index 8f57e2e8..3d94ac8f 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -242,15 +242,16 @@ engines: disabled: True - name : gitlab - engine : xpath + engine : json_engine paging : True - search_url : https://gitlab.com/search?page={pageno}&search={query} - url_xpath : //li[@class="project-row"]//a[@class="project"]/@href - title_xpath : //li[@class="project-row"]//span[contains(@class, "project-full-name")] - content_xpath : //li[@class="project-row"]//div[@class="description"]/p + search_url : https://gitlab.com/api/v4/projects?search={query}&page={pageno} + url_query : web_url + title_query : name_with_namespace + content_query : description + page_size : 20 categories : it shortcut : gl - timeout : 5.0 + timeout : 10.0 disabled : True - name : github