diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 56c6b36c..5de461cf 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -1,8 +1,9 @@ ## Bing (Web) -# +# # @website https://www.bing.com -# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month -# +# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), +# max. 5000 query/month +# # @using-api no (because of query limit) # @results HTML (using search portal) # @stable no (HTML can change) @@ -58,8 +59,8 @@ def response(resp): content = escape(' '.join(result.xpath('.//p//text()'))) # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'content': content}) # return results if something is found @@ -74,8 +75,8 @@ def response(resp): content = escape(' '.join(result.xpath('.//p//text()'))) # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'content': content}) # return results diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index b3eabba4..6c5e49bc 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -1,17 +1,19 @@ ## Bing (Images) -# +# # @website https://www.bing.com/images -# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month -# +# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), +# max. 5000 query/month +# # @using-api no (because of query limit) # @results HTML (using search portal) # @stable no (HTML can change) # @parse url, title, img_src # -# @todo currently there are up to 35 images receive per page, because bing does not parse count=10. limited response to 10 images +# @todo currently there are up to 35 images receive per page, +# because bing does not parse count=10. +# limited response to 10 images from urllib import urlencode -from cgi import escape from lxml import html from yaml import load import re @@ -51,15 +53,15 @@ def response(resp): dom = html.fromstring(resp.content) # init regex for yaml-parsing - p = re.compile( '({|,)([a-z]+):(")') + p = re.compile('({|,)([a-z]+):(")') # parse results for result in dom.xpath('//div[@class="dg_u"]'): link = result.xpath('./a')[0] # parse yaml-data (it is required to add a space, to make it parsable) - yaml_data = load(p.sub( r'\1\2: \3', link.attrib.get('m'))) - + yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m'))) + title = link.attrib.get('t1') #url = 'http://' + link.attrib.get('t3') url = yaml_data.get('surl') @@ -69,7 +71,7 @@ def response(resp): results.append({'template': 'images.html', 'url': url, 'title': title, - 'content': '', + 'content': '', 'img_src': img_src}) # TODO stop parsing if 10 images are found diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 279f0d69..5dce4a2b 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -1,8 +1,9 @@ ## Bing (News) -# +# # @website https://www.bing.com/news -# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month -# +# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), +# max. 5000 query/month +# # @using-api no (because of query limit) # @results HTML (using search portal) # @stable no (HTML can change) @@ -57,12 +58,12 @@ def response(resp): url = link.attrib.get('href') title = ' '.join(link.xpath('.//text()')) contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()') - if contentXPath != None: + if contentXPath is not None: content = escape(' '.join(contentXPath)) - + # parse publishedDate publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()') - if publishedDateXPath != None: + if publishedDateXPath is not None: publishedDate = escape(' '.join(publishedDateXPath)) if re.match("^[0-9]+ minute(s|) ago$", publishedDate): @@ -89,10 +90,10 @@ def response(resp): except TypeError: # FIXME publishedDate = datetime.now() - + # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'publishedDate': publishedDate, 'content': content}) diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py index b5f0953d..d8841c1d 100644 --- a/searx/engines/currency_convert.py +++ b/searx/engines/currency_convert.py @@ -55,6 +55,6 @@ def response(resp): resp.search_params['to'].lower() ) - results.append({'answer' : answer, 'url': url}) + results.append({'answer': answer, 'url': url}) return results diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py index 75c2e507..a5bffa86 100644 --- a/searx/engines/dailymotion.py +++ b/searx/engines/dailymotion.py @@ -1,8 +1,8 @@ ## Dailymotion (Videos) -# +# # @website https://www.dailymotion.com # @provide-api yes (http://www.dailymotion.com/developer) -# +# # @using-api yes # @results JSON # @stable yes @@ -12,7 +12,6 @@ from urllib import urlencode from json import loads -from lxml import html # engine dependent config categories = ['videos'] diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index ff5e1d46..d436e816 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -1,8 +1,8 @@ ## Deviantart (Images) -# +# # @website https://www.deviantart.com/ # @provide-api yes (https://www.deviantart.com/developers/) (RSS) -# +# # @using-api no (TODO, rewrite to api) # @results HTML # @stable no (HTML can change) diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 296dd9b2..583e33f7 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -1,15 +1,17 @@ ## DuckDuckGo (Web) -# +# # @website https://duckduckgo.com/ -# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site -# +# @provide-api yes (https://duckduckgo.com/api), +# but not all results from search-site +# # @using-api no # @results HTML (using search portal) # @stable no (HTML can change) # @parse url, title, content # # @todo rewrite to api -# @todo language support (the current used site does not support language-change) +# @todo language support +# (the current used site does not support language-change) from urllib import urlencode from lxml.html import fromstring @@ -37,7 +39,7 @@ def request(query, params): if params['language'] == 'all': locale = 'en-us' else: - locale = params['language'].replace('_','-').lower() + locale = params['language'].replace('_', '-').lower() params['url'] = url.format( query=urlencode({'q': query, 'kl': locale}), diff --git a/searx/engines/dummy.py b/searx/engines/dummy.py index 5a2cdf6b..c60b7a5d 100644 --- a/searx/engines/dummy.py +++ b/searx/engines/dummy.py @@ -1,5 +1,5 @@ ## Dummy -# +# # @results empty array # @stable yes diff --git a/searx/engines/faroo.py b/searx/engines/faroo.py index 8c69c5be..dada4758 100644 --- a/searx/engines/faroo.py +++ b/searx/engines/faroo.py @@ -1,8 +1,8 @@ ## Faroo (Web, News) -# +# # @website http://www.faroo.com # @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key -# +# # @using-api yes # @results JSON # @stable yes @@ -24,9 +24,10 @@ api_key = None url = 'http://www.faroo.com/' search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}' -search_category = {'general': 'web', +search_category = {'general': 'web', 'news': 'news'} + # do search-request def request(query, params): offset = (params['pageno']-1) * number_of_results + 1 @@ -48,7 +49,7 @@ def request(query, params): query=urlencode({'q': query}), language=language, categorie=categorie, - api_key=api_key ) + api_key=api_key) # using searx User-Agent params['headers']['User-Agent'] = searx_useragent() @@ -101,7 +102,7 @@ def response(resp): results.append({'template': 'images.html', 'url': result['url'], 'title': result['title'], - 'content': result['kwic'], + 'content': result['kwic'], 'img_src': result['iurl']}) # return results diff --git a/searx/engines/generalfile.py b/searx/engines/generalfile.py index 11d8b695..b7d71629 100644 --- a/searx/engines/generalfile.py +++ b/searx/engines/generalfile.py @@ -1,8 +1,8 @@ ## General Files (Files) -# +# # @website http://www.general-files.org # @provide-api no (nothing found) -# +# # @using-api no (because nothing found) # @results HTML (using search portal) # @stable no (HTML can change) diff --git a/searx/engines/github.py b/searx/engines/github.py index 53fec029..a68aed14 100644 --- a/searx/engines/github.py +++ b/searx/engines/github.py @@ -1,8 +1,8 @@ ## Github (It) -# +# # @website https://github.com/ # @provide-api yes (https://developer.github.com/v3/) -# +# # @using-api yes # @results JSON # @stable yes (using api) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 6c99f280..7b8b15b4 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -1,8 +1,9 @@ ## Google (Images) -# +# # @website https://www.google.com -# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! -# +# @provide-api yes (https://developers.google.com/web-search/docs/), +# deprecated! +# # @using-api yes # @results JSON # @stable yes (but deprecated) diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index becc7e21..eb114f9c 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -1,8 +1,9 @@ ## Google (News) -# +# # @website https://www.google.com -# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! -# +# @provide-api yes (https://developers.google.com/web-search/docs/), +# deprecated! +# # @using-api yes # @results JSON # @stable yes (but deprecated) diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py index f727ca8e..36b6011e 100644 --- a/searx/engines/openstreetmap.py +++ b/searx/engines/openstreetmap.py @@ -39,16 +39,16 @@ def response(resp): url = result_base_url.format(osm_type=osm_type, osm_id=r['osm_id']) - osm = {'type':osm_type, - 'id':r['osm_id']} + osm = {'type': osm_type, + 'id': r['osm_id']} - geojson = r.get('geojson') + geojson = r.get('geojson') # if no geojson is found and osm_type is a node, add geojson Point if not geojson and\ osm_type == 'node': - geojson = {u'type':u'Point', - u'coordinates':[r['lon'],r['lat']]} + geojson = {u'type': u'Point', + u'coordinates': [r['lon'], r['lat']]} address_raw = r.get('address') address = {} @@ -59,20 +59,20 @@ def response(resp): r['class'] == 'tourism' or\ r['class'] == 'leisure': if address_raw.get('address29'): - address = {'name':address_raw.get('address29')} + address = {'name': address_raw.get('address29')} else: - address = {'name':address_raw.get(r['type'])} + address = {'name': address_raw.get(r['type'])} # add rest of adressdata, if something is already found if address.get('name'): - address.update({'house_number':address_raw.get('house_number'), - 'road':address_raw.get('road'), - 'locality':address_raw.get('city', - address_raw.get('town', - address_raw.get('village'))), - 'postcode':address_raw.get('postcode'), - 'country':address_raw.get('country'), - 'country_code':address_raw.get('country_code')}) + address.update({'house_number': address_raw.get('house_number'), + 'road': address_raw.get('road'), + 'locality': address_raw.get('city', + address_raw.get('town', + address_raw.get('village'))), + 'postcode': address_raw.get('postcode'), + 'country': address_raw.get('country'), + 'country_code': address_raw.get('country_code')}) else: address = None diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py index 9533b629..14905dc8 100644 --- a/searx/engines/piratebay.py +++ b/searx/engines/piratebay.py @@ -1,8 +1,8 @@ ## Piratebay (Videos, Music, Files) -# +# # @website https://thepiratebay.se # @provide-api no (nothing found) -# +# # @using-api no # @results HTML (using search portal) # @stable yes (HTML can change) @@ -23,7 +23,7 @@ url = 'https://thepiratebay.se/' search_url = url + 'search/{search_term}/{pageno}/99/{search_type}' # piratebay specific type-definitions -search_types = {'files': '0', +search_types = {'files': '0', 'music': '100', 'videos': '200'} diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py index aebea239..390e7ca1 100644 --- a/searx/engines/soundcloud.py +++ b/searx/engines/soundcloud.py @@ -1,8 +1,8 @@ ## Soundcloud (Music) -# +# # @website https://soundcloud.com # @provide-api yes (https://developers.soundcloud.com/) -# +# # @using-api yes # @results JSON # @stable yes diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py index edbe74a7..dcbb1890 100644 --- a/searx/engines/stackoverflow.py +++ b/searx/engines/stackoverflow.py @@ -1,8 +1,8 @@ ## Stackoverflow (It) -# +# # @website https://stackoverflow.com/ # @provide-api not clear (https://api.stackexchange.com/docs/advanced-search) -# +# # @using-api no # @results HTML # @stable no (HTML can change) @@ -50,8 +50,8 @@ def response(resp): content = escape(' '.join(result.xpath(content_xpath))) # append result - results.append({'url': href, - 'title': title, + results.append({'url': href, + 'title': title, 'content': content}) # return results diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py index 8de78144..0689150c 100644 --- a/searx/engines/twitter.py +++ b/searx/engines/twitter.py @@ -1,8 +1,8 @@ ## Twitter (Social media) -# +# # @website https://www.bing.com/news # @provide-api yes (https://dev.twitter.com/docs/using-search) -# +# # @using-api no # @results HTML (using search portal) # @stable no (HTML can change) diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py index 2a91e76f..c66c4148 100644 --- a/searx/engines/vimeo.py +++ b/searx/engines/vimeo.py @@ -1,8 +1,9 @@ ## Vimeo (Videos) -# +# # @website https://vimeo.com/ -# @provide-api yes (http://developer.vimeo.com/api), they have a maximum count of queries/hour -# +# @provide-api yes (http://developer.vimeo.com/api), +# they have a maximum count of queries/hour +# # @using-api no (TODO, rewrite to api) # @results HTML (using search portal) # @stable no (HTML can change) @@ -35,11 +36,12 @@ publishedDate_xpath = './/p[@class="meta"]//attribute::datetime' # do search-request def request(query, params): - params['url'] = search_url.format(pageno=params['pageno'] , + params['url'] = search_url.format(pageno=params['pageno'], query=urlencode({'q': query})) # TODO required? - params['cookies']['__utma'] = '00000000.000#0000000.0000000000.0000000000.0000000000.0' + params['cookies']['__utma'] =\ + '00000000.000#0000000.0000000000.0000000000.0000000000.0' return params diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py index 2345b24f..3ee0e91c 100644 --- a/searx/engines/yacy.py +++ b/searx/engines/yacy.py @@ -1,8 +1,9 @@ ## Yacy (Web, Images, Videos, Music, Files) -# +# # @website http://yacy.net -# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) -# +# @provide-api yes +# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) +# # @using-api yes # @results JSON # @stable yes @@ -16,7 +17,7 @@ from urllib import urlencode from dateutil import parser # engine dependent config -categories = ['general', 'images'] #TODO , 'music', 'videos', 'files' +categories = ['general', 'images'] # TODO , 'music', 'videos', 'files' paging = True language_support = True number_of_results = 5 @@ -28,7 +29,7 @@ search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limi # yacy specific type-definitions search_types = {'general': 'text', 'images': 'image', - 'files': 'app', + 'files': 'app', 'music': 'audio', 'videos': 'video'} diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 5e34a2b0..938540ec 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -1,8 +1,9 @@ ## Yahoo (Web) -# +# # @website https://search.yahoo.com/web -# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries -# +# @provide-api yes (https://developer.yahoo.com/boss/search/), +# $0.80/1000 queries +# # @using-api no (because pricing) # @results HTML (using search portal) # @stable no (HTML can change) @@ -40,8 +41,8 @@ def parse_url(url_string): if endpos > -1: endpositions.append(endpos) - if start==0 or len(endpositions) == 0: - return url_string + if start == 0 or len(endpositions) == 0: + return url_string else: end = min(endpositions) return unquote(url_string[start:end]) @@ -84,8 +85,8 @@ def response(resp): content = extract_text(result.xpath(content_xpath)[0]) # append result - results.append({'url': url, - 'title': title, + results.append({'url': url, + 'title': title, 'content': content}) # if no suggestion found, return results diff --git a/searx/engines/youtube.py b/searx/engines/youtube.py index 7d1c207f..973e799f 100644 --- a/searx/engines/youtube.py +++ b/searx/engines/youtube.py @@ -1,8 +1,8 @@ ## Youtube (Videos) -# +# # @website https://www.youtube.com/ # @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/) -# +# # @using-api yes # @results JSON # @stable yes