From aa85045a7d24ccc672c33a5f83ee92caa0386596 Mon Sep 17 00:00:00 2001 From: Pydo Date: Mon, 5 Sep 2016 14:50:26 -0400 Subject: [PATCH 1/6] Added seedpeer unitests --- tests/unit/engines/test_seedpeer.py | 159 ++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/unit/engines/test_seedpeer.py diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py new file mode 100644 index 00000000..379282d1 --- /dev/null +++ b/tests/unit/engines/test_seedpeer.py @@ -0,0 +1,159 @@ +import mock +from collections import defaultdict +from searx.engines import seedpeer +from searx.testing import SearxTestCase +from datetime import datetime + + +class TestSeedPeerEngine(SearxTestCase): + html = """ + + + + + + + +
+

Verified Narcos season 2 torrents

+ + comments | + verified + + Torrent nameAgeSizeSeedsPeersHealth
Narcos season 2 Full Version20 hours681.3 MB28 654 Health
Narcos season 2 Trusted Source12 hours787.1 MB64 220 Health
Full Narcos season 2 Download Usenet24 hours775.5 MB60 236 Health
Narcos season 2 2014 - DIRECT STREAMING Movies17 hours654.1 MB2 391 Health
Narcos season 2 2014 Movies20 hours754.5 MB21 919 Health


Search Binaries

 2 Narcos season 2 Torrents were found

+ + comments | + verified + + Torrent nameAgeSizeSeedsPeersHealth
Add to FacebookNarcos Season 2 Complete 720p WebRip EN-SUB x264-[MULVAcoded] S02 19 hours4.39 GB715 183 Health
Add to FacebookNarcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG 1 day2.48 GB861 332 Health

Related searches for: Narcos season 2


Other suggested searches:

Search for "narcos-season-2" on Torrentz2.eu
Search for "narcos-season-2" on Torrent-Finder
  
+
+ + + """ + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = seedpeer.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('seedpeer.eu', params['url']) + + def test_response_raises_attr_error_on_empty_response(self): + self.assertRaises(AttributeError, seedpeer.response, None) + self.assertRaises(AttributeError, seedpeer.response, []) + self.assertRaises(AttributeError, seedpeer.response, '') + self.assertRaises(AttributeError, seedpeer.response, '[]') + + def test_response_returns_empty_list(self): + response = mock.Mock(text='') + self.assertEqual(seedpeer.response(response), []) + + def test_response_returns_all_results(self): + response = mock.Mock(text=self.html) + results = seedpeer.response(response) + self.assertTrue(isinstance(results, list)) + self.assertEqual(len(results), 2) + + def test_response_returns_correct_results(self): + response = mock.Mock(text=self.html) + results = seedpeer.response(response) + self.assertEqual( + results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG ' + ) + self.assertEqual( + results[0]['url'], + 'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html' + ) + self.assertEqual(results[0]['content'], '2.48 GB, 1 day') + self.assertEqual(results[0]['seed'], '861') + self.assertEqual(results[0]['leech'], '332') \ No newline at end of file From 2c2123b2e8f87178dadbf82b2cefbcef483c41a7 Mon Sep 17 00:00:00 2001 From: Pydo Date: Mon, 5 Sep 2016 14:51:02 -0400 Subject: [PATCH 2/6] Added seepeer to config and added seepeer search parser --- searx/engines/seedpeer.py | 78 +++++++++++++++++++++++++++++++++++++++ searx/settings.yml | 4 ++ 2 files changed, 82 insertions(+) create mode 100644 searx/engines/seedpeer.py diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py new file mode 100644 index 00000000..854ebba0 --- /dev/null +++ b/searx/engines/seedpeer.py @@ -0,0 +1,78 @@ +# Seedpeer (Videos, Music, Files) +# +# @website http://seedpeer.eu +# @provide-api no (nothing found) +# +# @using-api no +# @results HTML (using search portal) +# @stable yes (HTML can change) +# @parse url, title, content, seed, leech, magnetlink + +from urlparse import urljoin +from cgi import escape +from urllib import quote +from lxml import html +from operator import itemgetter +from searx.engines.xpath import extract_text + + +url = 'http://www.seedpeer.eu/' +search_url = url + 'search/{search_term}/7/{page_no}.html' +# specific xpath variables +torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a' +alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a' +title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()' +alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a' +seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()' +alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()' +peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()' +alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()' +age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()' +alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()' +size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()' +alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(search_term=quote(query), + page_no=params['pageno'] - 1) + return params + + +# get response from search-request +def response(resp): + results = [] + dom = html.fromstring(resp.text) + torrent_links = dom.xpath(torrent_xpath) + if len(torrent_links) > 0: + seeds = dom.xpath(seeds_xpath) + peers = dom.xpath(peers_xpath) + titles = dom.xpath(title_xpath) + sizes = dom.xpath(size_xpath) + ages = dom.xpath(age_xpath) + else: # under ~5 results uses a different xpath + torrent_links = dom.xpath(alternative_torrent_xpath) + seeds = dom.xpath(alternative_seeds_xpath) + peers = dom.xpath(alternative_peers_xpath) + titles = dom.xpath(alternative_title_xpath) + sizes = dom.xpath(alternative_size_xpath) + ages = dom.xpath(alternative_age_xpath) + # return empty array if nothing is found + if not torrent_links: + return [] + + # parse results + for index, result in enumerate(torrent_links): + link = result.attrib.get('href') + href = urljoin(url, link) + results.append({'url': href, + 'title': titles[index].text_content(), + 'content': '{}, {}'.format(sizes[index], ages[index]), + 'seed': seeds[index], + 'leech': peers[index], + + 'template': 'torrent.html'}) + + # return results sorted by seeder + return sorted(results, key=itemgetter('seed'), reverse=True) diff --git a/searx/settings.yml b/searx/settings.yml index 2c034a86..570ad1f8 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -495,6 +495,10 @@ engines: timeout: 6.0 categories : science + - name : seedpeer + engine : seedpeer + shortcut: speu + #The blekko technology and team have joined IBM Watson! -> https://blekko.com/ # - name : blekko images # engine : blekko_images From ec4a03628edfed2ceb192a323a47b2bc6271bdcc Mon Sep 17 00:00:00 2001 From: Pydo Date: Mon, 5 Sep 2016 15:37:20 -0400 Subject: [PATCH 3/6] Put html fixture in file to be pep8 line length compliant --- tests/unit/engines/seedpeer_fixture.html | 110 +++++++++++++++++++++ tests/unit/engines/test_seedpeer.py | 116 +---------------------- 2 files changed, 114 insertions(+), 112 deletions(-) create mode 100644 tests/unit/engines/seedpeer_fixture.html diff --git a/tests/unit/engines/seedpeer_fixture.html b/tests/unit/engines/seedpeer_fixture.html new file mode 100644 index 00000000..28207bfa --- /dev/null +++ b/tests/unit/engines/seedpeer_fixture.html @@ -0,0 +1,110 @@ + + + + + + + +
+

Verified Narcos season 2 torrents

+ + comments | + verified + + Torrent nameAgeSizeSeedsPeersHealth
Narcos season 2 Full Version20 hours681.3 MB28 654 Health
Narcos season 2 Trusted Source12 hours787.1 MB64 220 Health
Full Narcos season 2 Download Usenet24 hours775.5 MB60 236 Health
Narcos season 2 2014 - DIRECT STREAMING Movies17 hours654.1 MB2 391 Health
Narcos season 2 2014 Movies20 hours754.5 MB21 919 Health


Search Binaries

 2 Narcos season 2 Torrents were found

+ + comments | + verified + + Torrent nameAgeSizeSeedsPeersHealth
Add to FacebookNarcos Season 2 Complete 720p WebRip EN-SUB x264-[MULVAcoded] S02 19 hours4.39 GB715 183 Health
Add to FacebookNarcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG 1 day2.48 GB861 332 Health

Related searches for: Narcos season 2


Other suggested searches:

Search for "narcos-season-2" on Torrentz2.eu
Search for "narcos-season-2" on Torrent-Finder
  
+
+ + \ No newline at end of file diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py index 379282d1..37b2de8e 100644 --- a/tests/unit/engines/test_seedpeer.py +++ b/tests/unit/engines/test_seedpeer.py @@ -6,118 +6,10 @@ from datetime import datetime class TestSeedPeerEngine(SearxTestCase): - html = """ - - - - - - - -
-

Verified Narcos season 2 torrents

- - comments | - verified - - Torrent nameAgeSizeSeedsPeersHealth
Narcos season 2 Full Version20 hours681.3 MB28 654 Health
Narcos season 2 Trusted Source12 hours787.1 MB64 220 Health
Full Narcos season 2 Download Usenet24 hours775.5 MB60 236 Health
Narcos season 2 2014 - DIRECT STREAMING Movies17 hours654.1 MB2 391 Health
Narcos season 2 2014 Movies20 hours754.5 MB21 919 Health


Search Binaries

 2 Narcos season 2 Torrents were found

- - comments | - verified - - Torrent nameAgeSizeSeedsPeersHealth
Add to FacebookNarcos Season 2 Complete 720p WebRip EN-SUB x264-[MULVAcoded] S02 19 hours4.39 GB715 183 Health
Add to FacebookNarcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG 1 day2.48 GB861 332 Health

Related searches for: Narcos season 2


Other suggested searches:

Search for "narcos-season-2" on Torrentz2.eu
Search for "narcos-season-2" on Torrent-Finder
  
-
- - - """ + html = '' + with open('./tests/unit/engines/seedpeer_fixture.html') as fixture: + html += fixture.read() def test_request(self): query = 'test_query' @@ -156,4 +48,4 @@ class TestSeedPeerEngine(SearxTestCase): ) self.assertEqual(results[0]['content'], '2.48 GB, 1 day') self.assertEqual(results[0]['seed'], '861') - self.assertEqual(results[0]['leech'], '332') \ No newline at end of file + self.assertEqual(results[0]['leech'], '332') From 6f87bf2a1c76f1b94ad2119df7fb938c2307e370 Mon Sep 17 00:00:00 2001 From: Pydo Date: Sat, 1 Oct 2016 10:28:01 -0400 Subject: [PATCH 4/6] Disabled seepeer.eu by default since it does not support https --- searx/settings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/searx/settings.yml b/searx/settings.yml index 570ad1f8..b13a3833 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -498,6 +498,7 @@ engines: - name : seedpeer engine : seedpeer shortcut: speu + disabled: True #The blekko technology and team have joined IBM Watson! -> https://blekko.com/ # - name : blekko images From ccd1d9389139d5f459a1496e6e26c92e43ffc7c1 Mon Sep 17 00:00:00 2001 From: Pydo Date: Sat, 1 Oct 2016 11:04:14 -0400 Subject: [PATCH 5/6] Add pydo to authors --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 505b28ee..f7864887 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -58,3 +58,4 @@ generally made searx better: - marc @a01200356 - Harry Wood @harry-wood - Thomas Renard @threnard +- Pydo ``_ From 01844b6f04c99a7a5219ec06afe4266c76a57aeb Mon Sep 17 00:00:00 2001 From: Pydo Date: Sat, 1 Oct 2016 19:22:36 -0400 Subject: [PATCH 6/6] Set default categories for seedpeer provider --- searx/settings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/searx/settings.yml b/searx/settings.yml index 2846b2a0..f6848a24 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -498,6 +498,7 @@ engines: - name : seedpeer engine : seedpeer shortcut: speu + categories: files, music, videos disabled: True - name : dictzone