Merge pull request #677 from pydo/feature/seedpeer-engine-integration

Feature/seedpeer engine integration
2016-10-03 13:21:12 +02:00 · 2016-10-03 13:21:12 +02:00 · e7d005c621
parent dceb903114 beff8c6007
commit e7d005c621
5 changed files with 246 additions and 0 deletions
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@ -58,3 +58,4 @@ generally made searx better:
 - marc @a01200356
 - Harry Wood @harry-wood
 - Thomas Renard @threnard
+- Pydo `<https://github.com/pydo>`_
--- a/searx/engines/seedpeer.py
+++ b/searx/engines/seedpeer.py
@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     http://seedpeer.eu
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import quote
+from lxml import html
+from operator import itemgetter
+from searx.engines.xpath import extract_text
+
+
+url = 'http://www.seedpeer.eu/'
+search_url = url + 'search/{search_term}/7/{page_no}.html'
+# specific xpath variables
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'] - 1)
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    torrent_links = dom.xpath(torrent_xpath)
+    if len(torrent_links) > 0:
+        seeds = dom.xpath(seeds_xpath)
+        peers = dom.xpath(peers_xpath)
+        titles = dom.xpath(title_xpath)
+        sizes = dom.xpath(size_xpath)
+        ages = dom.xpath(age_xpath)
+    else:  # under ~5 results uses a different xpath
+        torrent_links = dom.xpath(alternative_torrent_xpath)
+        seeds = dom.xpath(alternative_seeds_xpath)
+        peers = dom.xpath(alternative_peers_xpath)
+        titles = dom.xpath(alternative_title_xpath)
+        sizes = dom.xpath(alternative_size_xpath)
+        ages = dom.xpath(alternative_age_xpath)
+    # return empty array if nothing is found
+    if not torrent_links:
+        return []
+
+    # parse results
+    for index, result in enumerate(torrent_links):
+        link = result.attrib.get('href')
+        href = urljoin(url, link)
+        results.append({'url': href,
+                        'title': titles[index].text_content(),
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
+                        'seed': seeds[index],
+                        'leech': peers[index],
+
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)
--- a/searx/settings.yml
+++ b/searx/settings.yml
@ -495,6 +495,12 @@ engines:
    timeout: 6.0
    categories : science

+  - name : seedpeer
+    engine : seedpeer
+    shortcut: speu
+    categories: files, music, videos
+    disabled: True
+
  - name : dictzone
    engine : dictzone
    shortcut : dc
--- a/tests/unit/engines/seedpeer_fixture.html
+++ b/tests/unit/engines/seedpeer_fixture.html
@ -0,0 +1,110 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml">
+        <head>
+        </head>
+        <body>
+            <div id="header">
+                <div id="whoIsYou">
+                        <a href="/lang.php"><small>SeedPeer in your own language?</small></a>&nbsp;&nbsp;&nbsp;<a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
+                        </div>
+
+                <script type="text/javascript">
+                whoIsYou();
+                </script>
+                    <div id="search">
+                        <form action="/search.php" method="get">
+                            <input id="topsearchbar" name="search" value="narcos season 2" />
+                            <input type="submit" class="searchbutton" value="Torrents" />
+                            <input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
+                        </form>
+                        <div id="suggestion"></div>
+                    </div>
+                    <div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
+                    <div id="subtext"><a href="/">Home</a> &gt; <a href="/search.html">Torrent search</a> &gt; Narcos season 2 | page 1</div>
+            </div>
+        <div id="nav">
+            <ul>
+        <!--
+                <li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
+
+        -->
+                <li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
+                    <ul>
+                        <li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
+                        <li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
+                        <li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
+                        <li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
+                        <li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
+                        <li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
+                        <li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
+                    </ul>
+                    </li>
+                        <li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
+                        <li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
+                        <li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
+                        <li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
+                        <li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
+                    <ul>
+                        <li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
+                        <li><a title="Contact Us" href="/contact.html">Contact</a></li>
+                        <li><a title="Frequently  Asked Questions" href="/faq.html">FAQ</a></li>
+                        <li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
+                        <li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
+                    </ul>
+                    </li>
+                    <!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
+                </ul>
+            <div class="clear"></div>
+        </div>
+        <div id="body"><div id="pageTop"></div>
+        <div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
+        <span style="float:right">
+        <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
+        <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
+        </span>
+        <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a>&nbsp;2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
+        <span style="float:right">
+        <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
+        <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
+        </span>
+        <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
+        <table width="100%">
+        <tr>
+        <td width="30%">
+        <h2>Torrents Download</h2>
+        <a href="/">Torrent search</a><br />
+        <a href="/browse.html">Browse categories</a><br />
+        <a href="/verified.html">Verified Torrents</a><br />
+        <a href="/order-date.html">Today's torrents</a><br />
+        <a href="/yesterday.html">Yesterday's torrents</a><br />
+        <a href="/stats.html">Statistics</a><br />
+        <br />
+        <a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
+        </td>
+        <td width="30%"><h2>Cool Stuff</h2>
+        <a href="/promotional.php">Promotional</a><br />
+        <a href="/contact.html">Advertising Information</a><br />
+        <strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
+        <a href="http://www.utorrent.com">&micro;Torrent Client</a><br />
+        <a href="/blog">Seedpeer Blog</a><br />
+        </td>
+        <td width="30%"><h2>Links</h2>
+         <a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
+         <a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
+         <a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
+         <a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
+         <a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
+         <a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
+         <a href="https://monova.org" target="_blank">Monova</a><br />
+         <a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
+         <a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
+        </td>
+        <td width="10%"><div id="bottomlogo"></div></td>
+        </tr>
+        </table>
+        <br />
+        <br />
+        </div>
+        </div>
+        </body>
+        </html>
--- a/tests/unit/engines/test_seedpeer.py
+++ b/tests/unit/engines/test_seedpeer.py
@ -0,0 +1,51 @@
+import mock
+from collections import defaultdict
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+from datetime import datetime
+
+
+class TestSeedPeerEngine(SearxTestCase):
+
+    html = ''
+    with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
+        html += fixture.read()
+
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['pageno'] = 1
+        params = seedpeer.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertIn(query, params['url'])
+        self.assertIn('seedpeer.eu', params['url'])
+
+    def test_response_raises_attr_error_on_empty_response(self):
+        self.assertRaises(AttributeError, seedpeer.response, None)
+        self.assertRaises(AttributeError, seedpeer.response, [])
+        self.assertRaises(AttributeError, seedpeer.response, '')
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+    def test_response_returns_empty_list(self):
+        response = mock.Mock(text='<html></html>')
+        self.assertEqual(seedpeer.response(response), [])
+
+    def test_response_returns_all_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertTrue(isinstance(results, list))
+        self.assertEqual(len(results), 2)
+
+    def test_response_returns_correct_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertEqual(
+            results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
+        )
+        self.assertEqual(
+            results[0]['url'],
+            'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
+        )
+        self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
+        self.assertEqual(results[0]['seed'], '861')
+        self.assertEqual(results[0]['leech'], '332')