Merge branch 'master' into feature/fix-config
This commit is contained in:
rachmadani haryono
GitHub
commit
ac357b12e3
8 changed files with 22 additions and 711 deletions
|
|
@ -11,9 +11,9 @@
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import date, timedelta
|
from datetime import date, timedelta
|
||||||
|
from json import loads
|
||||||
from lxml import html
|
from lxml import html
|
||||||
from searx.url_utils import urlencode, urlparse, parse_qs
|
from searx.url_utils import urlencode
|
||||||
|
|
||||||
|
|
||||||
# engine dependent config
|
# engine dependent config
|
||||||
categories = ['images']
|
categories = ['images']
|
||||||
|
|
@ -25,8 +25,7 @@ number_of_results = 100
|
||||||
search_url = 'https://www.google.com/search'\
|
search_url = 'https://www.google.com/search'\
|
||||||
'?{query}'\
|
'?{query}'\
|
||||||
'&tbm=isch'\
|
'&tbm=isch'\
|
||||||
'&gbv=1'\
|
'&yv=2'\
|
||||||
'&sa=G'\
|
|
||||||
'&{search_options}'
|
'&{search_options}'
|
||||||
time_range_attr = "qdr:{range}"
|
time_range_attr = "qdr:{range}"
|
||||||
time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
|
time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
|
||||||
|
|
@ -38,6 +37,7 @@ time_range_dict = {'day': 'd',
|
||||||
# do search-request
|
# do search-request
|
||||||
def request(query, params):
|
def request(query, params):
|
||||||
search_options = {
|
search_options = {
|
||||||
|
'ijn': params['pageno'] - 1,
|
||||||
'start': (params['pageno'] - 1) * number_of_results
|
'start': (params['pageno'] - 1) * number_of_results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -51,7 +51,7 @@ def request(query, params):
|
||||||
search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
|
search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
|
||||||
|
|
||||||
if safesearch and params['safesearch']:
|
if safesearch and params['safesearch']:
|
||||||
search_options['safe'] = 'active'
|
search_options['safe'] = 'on'
|
||||||
|
|
||||||
params['url'] = search_url.format(query=urlencode({'q': query}),
|
params['url'] = search_url.format(query=urlencode({'q': query}),
|
||||||
search_options=urlencode(search_options))
|
search_options=urlencode(search_options))
|
||||||
|
|
@ -61,30 +61,27 @@ def request(query, params):
|
||||||
|
|
||||||
# get response from search-request
|
# get response from search-request
|
||||||
def response(resp):
|
def response(resp):
|
||||||
|
results = []
|
||||||
|
|
||||||
dom = html.fromstring(resp.text)
|
dom = html.fromstring(resp.text)
|
||||||
|
|
||||||
results = []
|
# parse results
|
||||||
for element in dom.xpath('//div[@id="search"] //td'):
|
for result in dom.xpath('//div[contains(@class, "rg_meta")]/text()'):
|
||||||
link = element.xpath('./a')[0]
|
|
||||||
|
|
||||||
google_url = urlparse(link.xpath('.//@href')[0])
|
try:
|
||||||
query = parse_qs(google_url.query)
|
metadata = loads(result)
|
||||||
source_url = next(iter(query.get('q', [])), None)
|
img_format = "{0} {1}x{2}".format(metadata['ity'], str(metadata['ow']), str(metadata['oh']))
|
||||||
|
source = "{0} ({1})".format(metadata['st'], metadata['isu'])
|
||||||
|
results.append({'url': metadata['ru'],
|
||||||
|
'title': metadata['pt'],
|
||||||
|
'content': metadata['s'],
|
||||||
|
'source': source,
|
||||||
|
'img_format': img_format,
|
||||||
|
'thumbnail_src': metadata['tu'],
|
||||||
|
'img_src': metadata['ou'],
|
||||||
|
'template': 'images.html'})
|
||||||
|
|
||||||
title_parts = element.xpath('./cite//following-sibling::*/text()')
|
except:
|
||||||
title_parts.extend(element.xpath('./cite//following-sibling::text()')[:-1])
|
|
||||||
|
|
||||||
result = {
|
|
||||||
'title': ''.join(title_parts),
|
|
||||||
'content': '',
|
|
||||||
'template': 'images.html',
|
|
||||||
'url': source_url,
|
|
||||||
'img_src': source_url,
|
|
||||||
'thumbnail_src': next(iter(link.xpath('.//img //@src')), None)
|
|
||||||
}
|
|
||||||
|
|
||||||
if not source_url or not result['thumbnail_src']:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
results.append(result)
|
|
||||||
return results
|
return results
|
||||||
|
|
|
||||||
|
|
@ -1,75 +0,0 @@
|
||||||
# Seedpeer (Videos, Music, Files)
|
|
||||||
#
|
|
||||||
# @website http://seedpeer.eu
|
|
||||||
# @provide-api no (nothing found)
|
|
||||||
#
|
|
||||||
# @using-api no
|
|
||||||
# @results HTML (using search portal)
|
|
||||||
# @stable yes (HTML can change)
|
|
||||||
# @parse url, title, content, seed, leech, magnetlink
|
|
||||||
|
|
||||||
from lxml import html
|
|
||||||
from operator import itemgetter
|
|
||||||
from searx.url_utils import quote, urljoin
|
|
||||||
|
|
||||||
|
|
||||||
url = 'http://www.seedpeer.eu/'
|
|
||||||
search_url = url + 'search/{search_term}/7/{page_no}.html'
|
|
||||||
# specific xpath variables
|
|
||||||
torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
|
|
||||||
alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
|
|
||||||
title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
|
|
||||||
alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
|
|
||||||
seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
|
|
||||||
alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
|
|
||||||
peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
|
|
||||||
alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
|
|
||||||
age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
|
|
||||||
alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
|
|
||||||
size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
|
|
||||||
alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
|
|
||||||
|
|
||||||
|
|
||||||
# do search-request
|
|
||||||
def request(query, params):
|
|
||||||
params['url'] = search_url.format(search_term=quote(query),
|
|
||||||
page_no=params['pageno'] - 1)
|
|
||||||
return params
|
|
||||||
|
|
||||||
|
|
||||||
# get response from search-request
|
|
||||||
def response(resp):
|
|
||||||
results = []
|
|
||||||
dom = html.fromstring(resp.text)
|
|
||||||
torrent_links = dom.xpath(torrent_xpath)
|
|
||||||
if len(torrent_links) > 0:
|
|
||||||
seeds = dom.xpath(seeds_xpath)
|
|
||||||
peers = dom.xpath(peers_xpath)
|
|
||||||
titles = dom.xpath(title_xpath)
|
|
||||||
sizes = dom.xpath(size_xpath)
|
|
||||||
ages = dom.xpath(age_xpath)
|
|
||||||
else: # under ~5 results uses a different xpath
|
|
||||||
torrent_links = dom.xpath(alternative_torrent_xpath)
|
|
||||||
seeds = dom.xpath(alternative_seeds_xpath)
|
|
||||||
peers = dom.xpath(alternative_peers_xpath)
|
|
||||||
titles = dom.xpath(alternative_title_xpath)
|
|
||||||
sizes = dom.xpath(alternative_size_xpath)
|
|
||||||
ages = dom.xpath(alternative_age_xpath)
|
|
||||||
# return empty array if nothing is found
|
|
||||||
if not torrent_links:
|
|
||||||
return []
|
|
||||||
|
|
||||||
# parse results
|
|
||||||
for index, result in enumerate(torrent_links):
|
|
||||||
link = result.attrib.get('href')
|
|
||||||
href = urljoin(url, link)
|
|
||||||
results.append({'url': href,
|
|
||||||
'title': titles[index].text_content(),
|
|
||||||
'content': '{}, {}'.format(sizes[index], ages[index]),
|
|
||||||
'seed': seeds[index],
|
|
||||||
'leech': peers[index],
|
|
||||||
|
|
||||||
'template': 'torrent.html'})
|
|
||||||
|
|
||||||
# return results sorted by seeder
|
|
||||||
return sorted(results, key=itemgetter('seed'), reverse=True)
|
|
||||||
|
|
@ -1,86 +0,0 @@
|
||||||
"""
|
|
||||||
Subtitleseeker (Video)
|
|
||||||
|
|
||||||
@website http://www.subtitleseeker.com
|
|
||||||
@provide-api no
|
|
||||||
|
|
||||||
@using-api no
|
|
||||||
@results HTML
|
|
||||||
@stable no (HTML can change)
|
|
||||||
@parse url, title, content
|
|
||||||
"""
|
|
||||||
|
|
||||||
from lxml import html
|
|
||||||
from searx.languages import language_codes
|
|
||||||
from searx.engines.xpath import extract_text
|
|
||||||
from searx.url_utils import quote_plus
|
|
||||||
|
|
||||||
# engine dependent config
|
|
||||||
categories = ['videos']
|
|
||||||
paging = True
|
|
||||||
language = ""
|
|
||||||
|
|
||||||
# search-url
|
|
||||||
url = 'http://www.subtitleseeker.com/'
|
|
||||||
search_url = url + 'search/TITLES/{query}?p={pageno}'
|
|
||||||
|
|
||||||
# specific xpath variables
|
|
||||||
results_xpath = '//div[@class="boxRows"]'
|
|
||||||
|
|
||||||
|
|
||||||
# do search-request
|
|
||||||
def request(query, params):
|
|
||||||
params['url'] = search_url.format(query=quote_plus(query),
|
|
||||||
pageno=params['pageno'])
|
|
||||||
return params
|
|
||||||
|
|
||||||
|
|
||||||
# get response from search-request
|
|
||||||
def response(resp):
|
|
||||||
results = []
|
|
||||||
|
|
||||||
dom = html.fromstring(resp.text)
|
|
||||||
|
|
||||||
search_lang = ""
|
|
||||||
|
|
||||||
# dirty fix for languages named differenly in their site
|
|
||||||
if resp.search_params['language'][:2] == 'fa':
|
|
||||||
search_lang = 'Farsi'
|
|
||||||
elif resp.search_params['language'] == 'pt-BR':
|
|
||||||
search_lang = 'Brazilian'
|
|
||||||
elif resp.search_params['language'] != 'all':
|
|
||||||
search_lang = [lc[3]
|
|
||||||
for lc in language_codes
|
|
||||||
if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]
|
|
||||||
search_lang = search_lang[0].split(' (')[0]
|
|
||||||
|
|
||||||
# parse results
|
|
||||||
for result in dom.xpath(results_xpath):
|
|
||||||
link = result.xpath(".//a")[0]
|
|
||||||
href = link.attrib.get('href')
|
|
||||||
|
|
||||||
if language is not "":
|
|
||||||
href = href + language + '/'
|
|
||||||
elif search_lang:
|
|
||||||
href = href + search_lang + '/'
|
|
||||||
|
|
||||||
title = extract_text(link)
|
|
||||||
|
|
||||||
content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
|
|
||||||
content = content + " - "
|
|
||||||
text = extract_text(result.xpath('.//div[contains(@class,"grey-web")]')[0])
|
|
||||||
content = content + text
|
|
||||||
|
|
||||||
if result.xpath(".//span") != []:
|
|
||||||
content = content +\
|
|
||||||
" - (" +\
|
|
||||||
extract_text(result.xpath(".//span")) +\
|
|
||||||
")"
|
|
||||||
|
|
||||||
# append result
|
|
||||||
results.append({'url': href,
|
|
||||||
'title': title,
|
|
||||||
'content': content})
|
|
||||||
|
|
||||||
# return results
|
|
||||||
return results
|
|
||||||
|
|
@ -1,125 +0,0 @@
|
||||||
"""
|
|
||||||
Swisscows (Web, Images)
|
|
||||||
|
|
||||||
@website https://swisscows.ch
|
|
||||||
@provide-api no
|
|
||||||
|
|
||||||
@using-api no
|
|
||||||
@results HTML (using search portal)
|
|
||||||
@stable no (HTML can change)
|
|
||||||
@parse url, title, content
|
|
||||||
"""
|
|
||||||
|
|
||||||
from json import loads
|
|
||||||
import re
|
|
||||||
from lxml.html import fromstring
|
|
||||||
from searx.url_utils import unquote, urlencode
|
|
||||||
from searx.utils import match_language
|
|
||||||
|
|
||||||
# engine dependent config
|
|
||||||
categories = ['general', 'images']
|
|
||||||
paging = True
|
|
||||||
language_support = True
|
|
||||||
|
|
||||||
# search-url
|
|
||||||
base_url = 'https://swisscows.ch/'
|
|
||||||
search_string = '?{query}&page={page}'
|
|
||||||
|
|
||||||
supported_languages_url = base_url
|
|
||||||
|
|
||||||
# regex
|
|
||||||
regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')
|
|
||||||
regex_json_remove_start = re.compile(b'^initialData:\s*')
|
|
||||||
regex_json_remove_end = re.compile(b',\s*environment$')
|
|
||||||
regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
|
|
||||||
|
|
||||||
|
|
||||||
# do search-request
|
|
||||||
def request(query, params):
|
|
||||||
if params['language'] == 'all':
|
|
||||||
ui_language = 'browser'
|
|
||||||
region = 'browser'
|
|
||||||
else:
|
|
||||||
region = match_language(params['language'], supported_languages, language_aliases)
|
|
||||||
ui_language = region.split('-')[0]
|
|
||||||
|
|
||||||
search_path = search_string.format(
|
|
||||||
query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
|
|
||||||
page=params['pageno']
|
|
||||||
)
|
|
||||||
|
|
||||||
# image search query is something like 'image?{query}&page={page}'
|
|
||||||
if params['category'] == 'images':
|
|
||||||
search_path = 'image' + search_path
|
|
||||||
|
|
||||||
params['url'] = base_url + search_path
|
|
||||||
|
|
||||||
return params
|
|
||||||
|
|
||||||
|
|
||||||
# get response from search-request
|
|
||||||
def response(resp):
|
|
||||||
results = []
|
|
||||||
|
|
||||||
json_regex = regex_json.search(resp.text)
|
|
||||||
|
|
||||||
# check if results are returned
|
|
||||||
if not json_regex:
|
|
||||||
return []
|
|
||||||
|
|
||||||
json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))
|
|
||||||
json = loads(json_raw.decode('utf-8'))
|
|
||||||
|
|
||||||
# parse results
|
|
||||||
for result in json['Results'].get('items', []):
|
|
||||||
result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')
|
|
||||||
|
|
||||||
# parse image results
|
|
||||||
if result.get('ContentType', '').startswith('image'):
|
|
||||||
img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
|
|
||||||
|
|
||||||
# append result
|
|
||||||
results.append({'url': result['SourceUrl'],
|
|
||||||
'title': result['Title'],
|
|
||||||
'content': '',
|
|
||||||
'img_src': img_url,
|
|
||||||
'template': 'images.html'})
|
|
||||||
|
|
||||||
# parse general results
|
|
||||||
else:
|
|
||||||
result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
|
|
||||||
result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')
|
|
||||||
|
|
||||||
# append result
|
|
||||||
results.append({'url': result_url,
|
|
||||||
'title': result_title,
|
|
||||||
'content': result_content})
|
|
||||||
|
|
||||||
# parse images
|
|
||||||
for result in json.get('Images', []):
|
|
||||||
# decode image url
|
|
||||||
img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
|
|
||||||
|
|
||||||
# append result
|
|
||||||
results.append({'url': result['SourceUrl'],
|
|
||||||
'title': result['Title'],
|
|
||||||
'content': '',
|
|
||||||
'img_src': img_url,
|
|
||||||
'template': 'images.html'})
|
|
||||||
|
|
||||||
# return results
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
# get supported languages from their site
|
|
||||||
def _fetch_supported_languages(resp):
|
|
||||||
supported_languages = []
|
|
||||||
dom = fromstring(resp.text)
|
|
||||||
options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
|
|
||||||
for option in options:
|
|
||||||
code = option.xpath('./@data-search-language')[0]
|
|
||||||
if code.startswith('nb-'):
|
|
||||||
code = code.replace('nb', 'no', 1)
|
|
||||||
supported_languages.append(code)
|
|
||||||
|
|
||||||
return supported_languages
|
|
||||||
|
|
@ -568,24 +568,12 @@ engines:
|
||||||
engine : spotify
|
engine : spotify
|
||||||
shortcut : stf
|
shortcut : stf
|
||||||
|
|
||||||
- name : subtitleseeker
|
|
||||||
engine : subtitleseeker
|
|
||||||
shortcut : ss
|
|
||||||
# The language is an option. You can put any language written in english
|
|
||||||
# Examples : English, French, German, Hungarian, Chinese...
|
|
||||||
# language : English
|
|
||||||
|
|
||||||
- name : startpage
|
- name : startpage
|
||||||
engine : startpage
|
engine : startpage
|
||||||
shortcut : sp
|
shortcut : sp
|
||||||
timeout : 6.0
|
timeout : 6.0
|
||||||
disabled : True
|
disabled : True
|
||||||
|
|
||||||
- name : swisscows
|
|
||||||
engine : swisscows
|
|
||||||
shortcut : sw
|
|
||||||
disabled : True
|
|
||||||
|
|
||||||
- name : tokyotoshokan
|
- name : tokyotoshokan
|
||||||
engine : tokyotoshokan
|
engine : tokyotoshokan
|
||||||
shortcut : tt
|
shortcut : tt
|
||||||
|
|
@ -664,12 +652,6 @@ engines:
|
||||||
timeout: 6.0
|
timeout: 6.0
|
||||||
categories : science
|
categories : science
|
||||||
|
|
||||||
- name : seedpeer
|
|
||||||
engine : seedpeer
|
|
||||||
shortcut: speu
|
|
||||||
categories: files, music, videos
|
|
||||||
disabled: True
|
|
||||||
|
|
||||||
- name : dictzone
|
- name : dictzone
|
||||||
engine : dictzone
|
engine : dictzone
|
||||||
shortcut : dc
|
shortcut : dc
|
||||||
|
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
import mock
|
|
||||||
from collections import defaultdict
|
|
||||||
from searx.engines import seedpeer
|
|
||||||
from searx.testing import SearxTestCase
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
|
|
||||||
class TestSeedPeerEngine(SearxTestCase):
|
|
||||||
|
|
||||||
html = ''
|
|
||||||
with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
|
|
||||||
html += fixture.read()
|
|
||||||
|
|
||||||
def test_request(self):
|
|
||||||
query = 'test_query'
|
|
||||||
dicto = defaultdict(dict)
|
|
||||||
dicto['pageno'] = 1
|
|
||||||
params = seedpeer.request(query, dicto)
|
|
||||||
self.assertIn('url', params)
|
|
||||||
self.assertIn(query, params['url'])
|
|
||||||
self.assertIn('seedpeer.eu', params['url'])
|
|
||||||
|
|
||||||
def test_response_raises_attr_error_on_empty_response(self):
|
|
||||||
self.assertRaises(AttributeError, seedpeer.response, None)
|
|
||||||
self.assertRaises(AttributeError, seedpeer.response, [])
|
|
||||||
self.assertRaises(AttributeError, seedpeer.response, '')
|
|
||||||
self.assertRaises(AttributeError, seedpeer.response, '[]')
|
|
||||||
|
|
||||||
def test_response_returns_empty_list(self):
|
|
||||||
response = mock.Mock(text='<html></html>')
|
|
||||||
self.assertEqual(seedpeer.response(response), [])
|
|
||||||
|
|
||||||
def test_response_returns_all_results(self):
|
|
||||||
response = mock.Mock(text=self.html)
|
|
||||||
results = seedpeer.response(response)
|
|
||||||
self.assertTrue(isinstance(results, list))
|
|
||||||
self.assertEqual(len(results), 2)
|
|
||||||
|
|
||||||
def test_response_returns_correct_results(self):
|
|
||||||
response = mock.Mock(text=self.html)
|
|
||||||
results = seedpeer.response(response)
|
|
||||||
self.assertEqual(
|
|
||||||
results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
results[0]['url'],
|
|
||||||
'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
|
|
||||||
)
|
|
||||||
self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
|
|
||||||
self.assertEqual(results[0]['seed'], '861')
|
|
||||||
self.assertEqual(results[0]['leech'], '332')
|
|
||||||
|
|
@ -1,174 +0,0 @@
|
||||||
from collections import defaultdict
|
|
||||||
import mock
|
|
||||||
from searx.engines import subtitleseeker
|
|
||||||
from searx.testing import SearxTestCase
|
|
||||||
|
|
||||||
|
|
||||||
class TestSubtitleseekerEngine(SearxTestCase):
|
|
||||||
|
|
||||||
def test_request(self):
|
|
||||||
query = 'test_query'
|
|
||||||
dicto = defaultdict(dict)
|
|
||||||
dicto['pageno'] = 1
|
|
||||||
dicto['language'] = 'fr-FR'
|
|
||||||
params = subtitleseeker.request(query, dicto)
|
|
||||||
self.assertTrue('url' in params)
|
|
||||||
self.assertTrue(query in params['url'])
|
|
||||||
self.assertTrue('subtitleseeker.com' in params['url'])
|
|
||||||
|
|
||||||
def test_response(self):
|
|
||||||
dicto = defaultdict(dict)
|
|
||||||
dicto['language'] = 'fr-FR'
|
|
||||||
response = mock.Mock(search_params=dicto)
|
|
||||||
|
|
||||||
self.assertRaises(AttributeError, subtitleseeker.response, None)
|
|
||||||
self.assertRaises(AttributeError, subtitleseeker.response, [])
|
|
||||||
self.assertRaises(AttributeError, subtitleseeker.response, '')
|
|
||||||
self.assertRaises(AttributeError, subtitleseeker.response, '[]')
|
|
||||||
|
|
||||||
response = mock.Mock(text='<html></html>', search_params=dicto)
|
|
||||||
self.assertEqual(subtitleseeker.response(response), [])
|
|
||||||
|
|
||||||
html = """
|
|
||||||
<div class="boxRows">
|
|
||||||
<div class="boxRowsInner" style="width:600px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/movie.gif"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
<a href="http://this.is.the.url/"
|
|
||||||
class="blue" title="Title subtitle" >
|
|
||||||
This is the Title
|
|
||||||
</a>
|
|
||||||
<br><br>
|
|
||||||
<span class="f10b grey-dark arial" style="padding:0px 0px 5px 20px">
|
|
||||||
"Alternative Title"
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner f12b red" style="width:70px;">
|
|
||||||
1998
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f12" style="width:120px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/basket_put.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1039 Subs
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f10" style="width:130px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1 hours ago
|
|
||||||
</div>
|
|
||||||
<div class="clear"></div>
|
|
||||||
</div>
|
|
||||||
"""
|
|
||||||
response = mock.Mock(text=html, search_params=dicto)
|
|
||||||
results = subtitleseeker.response(response)
|
|
||||||
self.assertEqual(type(results), list)
|
|
||||||
self.assertEqual(len(results), 1)
|
|
||||||
self.assertEqual(results[0]['title'], 'This is the Title')
|
|
||||||
self.assertEqual(results[0]['url'], 'http://this.is.the.url/French/')
|
|
||||||
self.assertIn('1998', results[0]['content'])
|
|
||||||
self.assertIn('1039 Subs', results[0]['content'])
|
|
||||||
self.assertIn('Alternative Title', results[0]['content'])
|
|
||||||
|
|
||||||
dicto['language'] = 'pt-BR'
|
|
||||||
results = subtitleseeker.response(response)
|
|
||||||
self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')
|
|
||||||
|
|
||||||
html = """
|
|
||||||
<div class="boxRows">
|
|
||||||
<div class="boxRowsInner" style="width:600px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/movie.gif"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
<a href="http://this.is.the.url/"
|
|
||||||
class="blue" title="Title subtitle" >
|
|
||||||
This is the Title
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner f12b red" style="width:70px;">
|
|
||||||
1998
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f12" style="width:120px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/basket_put.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1039 Subs
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f10" style="width:130px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1 hours ago
|
|
||||||
</div>
|
|
||||||
<div class="clear"></div>
|
|
||||||
</div>
|
|
||||||
"""
|
|
||||||
dicto['language'] = 'all'
|
|
||||||
response = mock.Mock(text=html, search_params=dicto)
|
|
||||||
results = subtitleseeker.response(response)
|
|
||||||
self.assertEqual(type(results), list)
|
|
||||||
self.assertEqual(len(results), 1)
|
|
||||||
self.assertEqual(results[0]['title'], 'This is the Title')
|
|
||||||
self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
|
|
||||||
self.assertIn('1998', results[0]['content'])
|
|
||||||
self.assertIn('1039 Subs', results[0]['content'])
|
|
||||||
|
|
||||||
html = """
|
|
||||||
<div class="boxRows">
|
|
||||||
<div class="boxRowsInner" style="width:600px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/movie.gif"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
<a href="http://this.is.the.url/"
|
|
||||||
class="blue" title="Title subtitle" >
|
|
||||||
This is the Title
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner f12b red" style="width:70px;">
|
|
||||||
1998
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f12" style="width:120px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/basket_put.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1039 Subs
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f10" style="width:130px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1 hours ago
|
|
||||||
</div>
|
|
||||||
<div class="clear"></div>
|
|
||||||
</div>
|
|
||||||
"""
|
|
||||||
subtitleseeker.language = 'English'
|
|
||||||
response = mock.Mock(text=html, search_params=dicto)
|
|
||||||
results = subtitleseeker.response(response)
|
|
||||||
self.assertEqual(type(results), list)
|
|
||||||
self.assertEqual(len(results), 1)
|
|
||||||
self.assertEqual(results[0]['title'], 'This is the Title')
|
|
||||||
self.assertEqual(results[0]['url'], 'http://this.is.the.url/English/')
|
|
||||||
self.assertIn('1998', results[0]['content'])
|
|
||||||
self.assertIn('1039 Subs', results[0]['content'])
|
|
||||||
|
|
||||||
html = """
|
|
||||||
<div class="boxRowsInner" style="width:600px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/movie.gif"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
<a href="http://this.is.the.url/"
|
|
||||||
class="blue" title="Title subtitle" >
|
|
||||||
This is the Title
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner f12b red" style="width:70px;">
|
|
||||||
1998
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f12" style="width:120px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/basket_put.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1039 Subs
|
|
||||||
</div>
|
|
||||||
<div class="boxRowsInner grey-web f10" style="width:130px;">
|
|
||||||
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
|
|
||||||
style="width:16px; height:16px;" class="icon">
|
|
||||||
1 hours ago
|
|
||||||
</div>
|
|
||||||
"""
|
|
||||||
response = mock.Mock(text=html, search_params=dicto)
|
|
||||||
results = subtitleseeker.response(response)
|
|
||||||
self.assertEqual(type(results), list)
|
|
||||||
self.assertEqual(len(results), 0)
|
|
||||||
|
|
@ -1,157 +0,0 @@
|
||||||
from collections import defaultdict
|
|
||||||
import mock
|
|
||||||
from searx.engines import swisscows
|
|
||||||
from searx.testing import SearxTestCase
|
|
||||||
|
|
||||||
|
|
||||||
class TestSwisscowsEngine(SearxTestCase):
|
|
||||||
|
|
||||||
def test_request(self):
|
|
||||||
swisscows.supported_languages = ['de-AT', 'de-DE']
|
|
||||||
swisscows.language_aliases = {}
|
|
||||||
query = 'test_query'
|
|
||||||
dicto = defaultdict(dict)
|
|
||||||
dicto['pageno'] = 1
|
|
||||||
dicto['language'] = 'de-DE'
|
|
||||||
params = swisscows.request(query, dicto)
|
|
||||||
self.assertTrue('url' in params)
|
|
||||||
self.assertTrue(query in params['url'])
|
|
||||||
self.assertTrue('swisscows.ch' in params['url'])
|
|
||||||
self.assertTrue('uiLanguage=de' in params['url'])
|
|
||||||
self.assertTrue('region=de-DE' in params['url'])
|
|
||||||
|
|
||||||
dicto['language'] = 'all'
|
|
||||||
params = swisscows.request(query, dicto)
|
|
||||||
self.assertTrue('uiLanguage=browser' in params['url'])
|
|
||||||
self.assertTrue('region=browser' in params['url'])
|
|
||||||
|
|
||||||
dicto['category'] = 'images'
|
|
||||||
params = swisscows.request(query, dicto)
|
|
||||||
self.assertIn('image', params['url'])
|
|
||||||
|
|
||||||
def test_response(self):
|
|
||||||
self.assertRaises(AttributeError, swisscows.response, None)
|
|
||||||
self.assertRaises(AttributeError, swisscows.response, [])
|
|
||||||
self.assertRaises(AttributeError, swisscows.response, '')
|
|
||||||
self.assertRaises(AttributeError, swisscows.response, '[]')
|
|
||||||
|
|
||||||
response = mock.Mock(text=b'<html></html>')
|
|
||||||
self.assertEqual(swisscows.response(response), [])
|
|
||||||
|
|
||||||
response = mock.Mock(text=b'<html></html>')
|
|
||||||
self.assertEqual(swisscows.response(response), [])
|
|
||||||
|
|
||||||
html = b"""
|
|
||||||
<script>
|
|
||||||
App.Dispatcher.dispatch("initialize", {
|
|
||||||
html5history: true,
|
|
||||||
initialData: {"Request":
|
|
||||||
{"Page":1,
|
|
||||||
"ItemsCount":1,
|
|
||||||
"Query":"This should ",
|
|
||||||
"NormalizedQuery":"This should ",
|
|
||||||
"Region":"de-AT",
|
|
||||||
"UILanguage":"de"},
|
|
||||||
"Results":{"items":[
|
|
||||||
{"Title":"\uE000This should\uE001 be the title",
|
|
||||||
"Description":"\uE000This should\uE001 be the content.",
|
|
||||||
"Url":"http://this.should.be.the.link/",
|
|
||||||
"DisplayUrl":"www.\uE000this.should.be.the\uE001.link",
|
|
||||||
"Id":"782ef287-e439-451c-b380-6ebc14ba033d"},
|
|
||||||
{"Title":"Datei:This should1.svg",
|
|
||||||
"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png",
|
|
||||||
"SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should1.svg",
|
|
||||||
"DisplayUrl":"de.wikipedia.org/wiki/Datei:This should1.svg",
|
|
||||||
"Width":950,
|
|
||||||
"Height":534,
|
|
||||||
"FileSize":92100,
|
|
||||||
"ContentType":"image/jpeg",
|
|
||||||
"Thumbnail":{
|
|
||||||
"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png",
|
|
||||||
"ContentType":"image/jpeg",
|
|
||||||
"Width":300,
|
|
||||||
"Height":168,
|
|
||||||
"FileSize":9134},
|
|
||||||
"Id":"6a97a542-8f65-425f-b7f6-1178c3aba7be"
|
|
||||||
}
|
|
||||||
],"TotalCount":55300,
|
|
||||||
"Query":"This should "
|
|
||||||
},
|
|
||||||
"Images":[{"Title":"Datei:This should.svg",
|
|
||||||
"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png",
|
|
||||||
"SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should.svg",
|
|
||||||
"DisplayUrl":"de.wikipedia.org/wiki/Datei:This should.svg",
|
|
||||||
"Width":1280,
|
|
||||||
"Height":677,
|
|
||||||
"FileSize":50053,
|
|
||||||
"ContentType":"image/png",
|
|
||||||
"Thumbnail":{"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png",
|
|
||||||
"ContentType":"image/png",
|
|
||||||
"Width":300,
|
|
||||||
"Height":158,
|
|
||||||
"FileSize":8023},
|
|
||||||
"Id":"ae230fd8-a06a-47d6-99d5-e74766d8143a"}]},
|
|
||||||
environment: "production"
|
|
||||||
}).then(function (options) {
|
|
||||||
$('#Search_Form').on('submit', function (e) {
|
|
||||||
if (!Modernizr.history) return;
|
|
||||||
e.preventDefault();
|
|
||||||
|
|
||||||
var $form = $(this),
|
|
||||||
$query = $('#Query'),
|
|
||||||
query = $.trim($query.val()),
|
|
||||||
path = App.Router.makePath($form.attr('action'), null, $form.serializeObject())
|
|
||||||
|
|
||||||
if (query.length) {
|
|
||||||
options.html5history ?
|
|
||||||
ReactRouter.HistoryLocation.push(path) :
|
|
||||||
ReactRouter.RefreshLocation.push(path);
|
|
||||||
}
|
|
||||||
else $('#Query').trigger('blur');
|
|
||||||
});
|
|
||||||
|
|
||||||
});
|
|
||||||
</script>
|
|
||||||
"""
|
|
||||||
response = mock.Mock(text=html)
|
|
||||||
results = swisscows.response(response)
|
|
||||||
self.assertEqual(type(results), list)
|
|
||||||
self.assertEqual(len(results), 3)
|
|
||||||
self.assertEqual(results[0]['title'], 'This should be the title')
|
|
||||||
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
|
|
||||||
self.assertEqual(results[0]['content'], 'This should be the content.')
|
|
||||||
self.assertEqual(results[1]['title'], 'Datei:This should1.svg')
|
|
||||||
self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should1.svg')
|
|
||||||
self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should1.png')
|
|
||||||
self.assertEqual(results[1]['template'], 'images.html')
|
|
||||||
self.assertEqual(results[2]['title'], 'Datei:This should.svg')
|
|
||||||
self.assertEqual(results[2]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg')
|
|
||||||
self.assertEqual(results[2]['img_src'], 'http://ts2.mm.This/should.png')
|
|
||||||
self.assertEqual(results[2]['template'], 'images.html')
|
|
||||||
|
|
||||||
def test_fetch_supported_languages(self):
|
|
||||||
html = """<html></html>"""
|
|
||||||
response = mock.Mock(text=html)
|
|
||||||
languages = swisscows._fetch_supported_languages(response)
|
|
||||||
self.assertEqual(type(languages), list)
|
|
||||||
self.assertEqual(len(languages), 0)
|
|
||||||
|
|
||||||
html = """
|
|
||||||
<html>
|
|
||||||
<div id="regions-popup">
|
|
||||||
<div>
|
|
||||||
<ul>
|
|
||||||
<li><a data-search-language="browser"></a></li>
|
|
||||||
<li><a data-search-language="de-CH"></a></li>
|
|
||||||
<li><a data-search-language="fr-CH"></a></li>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</html>
|
|
||||||
"""
|
|
||||||
response = mock.Mock(text=html)
|
|
||||||
languages = swisscows._fetch_supported_languages(response)
|
|
||||||
self.assertEqual(type(languages), list)
|
|
||||||
self.assertEqual(len(languages), 3)
|
|
||||||
self.assertIn('de-CH', languages)
|
|
||||||
self.assertIn('fr-CH', languages)
|
|
||||||
Loading…
Reference in a new issue