Merge branch 'master' into searchpy2

searx/answerers/__init__.py

@@ -0,0 +1,46 @@
+from os import listdir
+from os.path import realpath, dirname, join, isdir
+from searx.utils import load_module
+from collections import defaultdict
+
+
+answerers_dir = dirname(realpath(__file__))
+
+
+def load_answerers():
+    answerers = []
+    for filename in listdir(answerers_dir):
+        if not isdir(join(answerers_dir, filename)):
+            continue
+        module = load_module('answerer.py', join(answerers_dir, filename))
+        if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
+            exit(2)
+        answerers.append(module)
+    return answerers
+
+
+def get_answerers_by_keywords(answerers):
+    by_keyword = defaultdict(list)
+    for answerer in answerers:
+        for keyword in answerer.keywords:
+            for keyword in answerer.keywords:
+                by_keyword[keyword].append(answerer.answer)
+    return by_keyword
+
+
+def ask(query):
+    results = []
+    query_parts = filter(None, query.query.split())
+
+    if query_parts[0] not in answerers_by_keywords:
+        return results
+
+    for answerer in answerers_by_keywords[query_parts[0]]:
+        result = answerer(query)
+        if result:
+            results.append(result)
+    return results
+
+
+answerers = load_answerers()
+answerers_by_keywords = get_answerers_by_keywords(answerers)

searx/answerers/random/answerer.py

@@ -0,0 +1,50 @@
+import random
+import string
+from flask_babel import gettext
+
+# required answerer attribute
+# specifies which search query keywords triggers this answerer
+keywords = ('random',)
+
+random_int_max = 2**31
+
+random_string_letters = string.lowercase + string.digits + string.uppercase
+
+
+def random_string():
+    return u''.join(random.choice(random_string_letters)
+                    for _ in range(random.randint(8, 32)))
+
+
+def random_float():
+    return unicode(random.random())
+
+
+def random_int():
+    return unicode(random.randint(-random_int_max, random_int_max))
+
+
+random_types = {u'string': random_string,
+                u'int': random_int,
+                u'float': random_float}
+
+
+# required answerer function
+# can return a list of results (any result type) for a given query
+def answer(query):
+    parts = query.query.split()
+    if len(parts) != 2:
+        return []
+
+    if parts[1] not in random_types:
+        return []
+
+    return [{'answer': random_types[parts[1]]()}]
+
+
+# required answerer function
+# returns information about the answerer
+def self_info():
+    return {'name': gettext('Random value generator'),
+            'description': gettext('Generate different random values'),
+            'examples': [u'random {}'.format(x) for x in random_types]}

searx/answerers/statistics/answerer.py

@@ -0,0 +1,51 @@
+from functools import reduce
+from operator import mul
+
+from flask_babel import gettext
+
+keywords = ('min',
+            'max',
+            'avg',
+            'sum',
+            'prod')
+
+
+# required answerer function
+# can return a list of results (any result type) for a given query
+def answer(query):
+    parts = query.query.split()
+
+    if len(parts) < 2:
+        return []
+
+    try:
+        args = map(float, parts[1:])
+    except:
+        return []
+
+    func = parts[0]
+    answer = None
+
+    if func == 'min':
+        answer = min(args)
+    elif func == 'max':
+        answer = max(args)
+    elif func == 'avg':
+        answer = sum(args) / len(args)
+    elif func == 'sum':
+        answer = sum(args)
+    elif func == 'prod':
+        answer = reduce(mul, args, 1)
+
+    if answer is None:
+        return []
+
+    return [{'answer': unicode(answer)}]
+
+
+# required answerer function
+# returns information about the answerer
+def self_info():
+    return {'name': gettext('Statistics functions'),
+            'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),
+            'examples': ['avg 123 548 2.04 24.2']}

searx/engines/__init__.py

@@ -16,13 +16,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-from os.path import realpath, dirname, splitext, join
+from os.path import realpath, dirname
 import sys
-from imp import load_source
 from flask_babel import gettext
 from operator import itemgetter
 from searx import settings
 from searx import logger
+from searx.utils import load_module
 
 
 logger = logger.getChild('engines')
@@ -32,6 +32,7 @@ engine_dir = dirname(realpath(__file__))
 engines = {}
 
 categories = {'general': []}
+_initialized = False
 
 engine_shortcuts = {}
 engine_default_args = {'paging': False,
@@ -46,16 +47,6 @@ engine_default_args = {'paging': False,
                        'time_range_support': False}
 
 
-def load_module(filename):
-    modname = splitext(filename)[0]
-    if modname in sys.modules:
-        del sys.modules[modname]
-    filepath = join(engine_dir, filename)
-    module = load_source(modname, filepath)
-    module.name = modname
-    return module
-
-
 def load_engine(engine_data):
 
     if '_' in engine_data['name']:
@@ -65,7 +56,7 @@ def load_engine(engine_data):
     engine_module = engine_data['engine']
 
     try:
-        engine = load_module(engine_module + '.py')
+        engine = load_module(engine_module + '.py', engine_dir)
     except:
         logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None

searx/engines/archlinux.py

@@ -12,7 +12,6 @@
 """
 
 from urlparse import urljoin
-from cgi import escape
 from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
@@ -135,7 +134,7 @@ def response(resp):
     for result in dom.xpath(xpath_results):
         link = result.xpath(xpath_link)[0]
         href = urljoin(base_url, link.attrib.get('href'))
-        title = escape(extract_text(link))
+        title = extract_text(link)
 
         results.append({'url': href,
                         'title': title})

searx/engines/base.py

@@ -16,7 +16,6 @@
 from lxml import etree
 from urllib import urlencode
 from searx.utils import searx_useragent
-from cgi import escape
 from datetime import datetime
 import re
 
@@ -94,7 +93,7 @@ def response(resp):
                 url = item.text
 
             elif item.attrib["name"] == "dcdescription":
-                content = escape(item.text[:300])
+                content = item.text[:300]
                 if len(item.text) > 300:
                     content += "..."
 

searx/engines/bing.py

@@ -14,7 +14,6 @@
 """
 
 from urllib import urlencode
-from cgi import escape
 from lxml import html
 from searx.engines.xpath import extract_text
 
@@ -32,18 +31,14 @@ search_string = 'search?{query}&first={offset}'
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    if params['language'] == 'all':
+    if params['language'] != 'all':
-        language = 'en-US'
+        query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),
-    else:
+                                         query.decode('utf-8')).encode('utf-8')
-        language = params['language'].replace('_', '-')
 
     search_path = search_string.format(
-        query=urlencode({'q': query, 'setmkt': language}),
+        query=urlencode({'q': query}),
         offset=offset)
 
-    params['cookies']['SRCHHPGUSR'] = \
-        'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]
-
     params['url'] = base_url + search_path
     return params
 
@@ -65,7 +60,7 @@ def response(resp):
         link = result.xpath('.//h3/a')[0]
         url = link.attrib.get('href')
         title = extract_text(link)
-        content = escape(extract_text(result.xpath('.//p')))
+        content = extract_text(result.xpath('.//p'))
 
         # append result
         results.append({'url': url,
@@ -77,7 +72,7 @@ def response(resp):
         link = result.xpath('.//h2/a')[0]
         url = link.attrib.get('href')
         title = extract_text(link)
-        content = escape(extract_text(result.xpath('.//p')))
+        content = extract_text(result.xpath('.//p'))
 
         # append result
         results.append({'url': url,

searx/engines/btdigg.py

@@ -11,7 +11,6 @@
 """
 
 from urlparse import urljoin
-from cgi import escape
 from urllib import quote
 from lxml import html
 from operator import itemgetter
@@ -51,8 +50,8 @@ def response(resp):
     for result in search_res:
         link = result.xpath('.//td[@class="torrent_name"]//a')[0]
         href = urljoin(url, link.attrib.get('href'))
-        title = escape(extract_text(link))
+        title = extract_text(link)
-        content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0]))
+        content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])
         content = "<br />".join(content.split("\n"))
 
         filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

searx/engines/dailymotion.py

@@ -14,7 +14,6 @@
 
 from urllib import urlencode
 from json import loads
-from cgi import escape
 from datetime import datetime
 
 # engine dependent config
@@ -57,7 +56,7 @@ def response(resp):
     for res in search_res['list']:
         title = res['title']
         url = res['url']
-        content = escape(res['description'])
+        content = res['description']
         thumbnail = res['thumbnail_360_url']
         publishedDate = datetime.fromtimestamp(res['created_time'], None)
         embedded = embedded_url.format(videoid=res['id'])

searx/engines/deezer.py

@@ -51,10 +51,11 @@ def response(resp):
             if url.startswith('http://'):
                 url = 'https' + url[4:]
 
-            content = result['artist']['name'] +\
+            content = '{} - {} - {}'.format(
-                " • " +\
+                result['artist']['name'],
-                result['album']['title'] +\
+                result['album']['title'],
-                " • " + result['title']
+                result['title'])
+
             embedded = embedded_url.format(audioid=result['id'])
 
             # append result

searx/engines/dictzone.py

@@ -12,7 +12,6 @@
 import re
 from urlparse import urljoin
 from lxml import html
-from cgi import escape
 from searx.utils import is_valid_lang
 
 categories = ['general']
@@ -62,8 +61,8 @@ def response(resp):
 
         results.append({
             'url': urljoin(resp.url, '?%d' % k),
-            'title': escape(from_result.text_content()),
+            'title': from_result.text_content(),
-            'content': escape('; '.join(to_results))
+            'content': '; '.join(to_results)
         })
 
     return results

searx/engines/digg.py

@@ -13,7 +13,6 @@
 from urllib import quote_plus
 from json import loads
 from lxml import html
-from cgi import escape
 from dateutil import parser
 
 # engine dependent config
@@ -56,7 +55,7 @@ def response(resp):
         url = result.attrib.get('data-contenturl')
         thumbnail = result.xpath('.//img')[0].attrib.get('src')
         title = ''.join(result.xpath(title_xpath))
-        content = escape(''.join(result.xpath(content_xpath)))
+        content = ''.join(result.xpath(content_xpath))
         pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')
         publishedDate = parser.parse(pubdate)
 

searx/engines/fdroid.py

@@ -9,7 +9,6 @@
  @parse        url, title, content
 """
 
-from cgi import escape
 from urllib import urlencode
 from searx.engines.xpath import extract_text
 from lxml import html
@@ -43,7 +42,7 @@ def response(resp):
         img_src = app.xpath('.//img/@src')[0]
 
         content = extract_text(app.xpath('./p')[0])
-        content = escape(content.replace(title, '', 1).strip())
+        content = content.replace(title, '', 1).strip()
 
         results.append({'url': url,
                         'title': title,

searx/engines/flickr.py

@@ -77,21 +77,13 @@ def response(resp):
 
         url = build_flickr_url(photo['owner'], photo['id'])
 
-        title = photo['title']
-
-        content = '<span class="photo-author">' +\
-                  photo['ownername'] +\
-                  '</span><br />' +\
-                  '<span class="description">' +\
-                  photo['description']['_content'] +\
-                  '</span>'
-
         # append result
         results.append({'url': url,
-                        'title': title,
+                        'title': photo['title'],
                         'img_src': img_src,
                         'thumbnail_src': thumbnail_src,
-                        'content': content,
+                        'content': photo['description']['_content'],
+                        'author': photo['ownername'],
                         'template': 'images.html'})
 
     # return results

searx/engines/flickr_noapi.py

@@ -102,16 +102,15 @@ def response(resp):
 
         title = photo.get('title', '')
 
-        content = '<span class="photo-author">' +\
+        author = photo['username']
-                  photo['username'] +\
-                  '</span><br />'
 
         # append result
         results.append({'url': url,
                         'title': title,
                         'img_src': img_src,
                         'thumbnail_src': thumbnail_src,
-                        'content': content,
+                        'content': '',
+                        'author': author,
                         'template': 'images.html'})
 
     return results

searx/engines/gigablast.py

@@ -10,7 +10,6 @@
  @parse       url, title, content
 """
 
-from cgi import escape
 from json import loads
 from random import randint
 from time import time
@@ -78,8 +77,8 @@ def response(resp):
     for result in response_json['results']:
         # append result
         results.append({'url': result['url'],
-                        'title': escape(result['title']),
+                        'title': result['title'],
-                        'content': escape(result['sum'])})
+                        'content': result['sum']})
 
     # return results
     return results

searx/engines/github.py

@@ -12,7 +12,6 @@
 
 from urllib import urlencode
 from json import loads
-from cgi import escape
 
 # engine dependent config
 categories = ['it']
@@ -48,7 +47,7 @@ def response(resp):
         url = res['html_url']
 
         if res['description']:
-            content = escape(res['description'][:500])
+            content = res['description'][:500]
         else:
             content = ''
 

searx/engines/google.py

@@ -9,7 +9,6 @@
 # @parse       url, title, content, suggestion
 
 import re
-from cgi import escape
 from urllib import urlencode
 from urlparse import urlparse, parse_qsl
 from lxml import html, etree
@@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname):
 def extract_text_from_dom(result, xpath):
     r = result.xpath(xpath)
     if len(r) > 0:
-        return escape(extract_text(r[0]))
+        return extract_text(r[0])
     return None
 
 
@@ -264,7 +263,7 @@ def response(resp):
     # parse suggestion
     for suggestion in dom.xpath(suggestion_xpath):
         # append suggestion
-        results.append({'suggestion': escape(extract_text(suggestion))})
+        results.append({'suggestion': extract_text(suggestion)})
 
     # return results
     return results

searx/engines/kickass.py

@@ -11,7 +11,6 @@
 """
 
 from urlparse import urljoin
-from cgi import escape
 from urllib import quote
 from lxml import html
 from operator import itemgetter
@@ -57,7 +56,7 @@ def response(resp):
         link = result.xpath('.//a[@class="cellMainLink"]')[0]
         href = urljoin(url, link.attrib['href'])
         title = extract_text(link)
-        content = escape(extract_text(result.xpath(content_xpath)))
+        content = extract_text(result.xpath(content_xpath))
         seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
         leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
         filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))

searx/engines/nyaa.py

@@ -9,7 +9,6 @@
  @parse        url, title, content, seed, leech, torrentfile
 """
 
-from cgi import escape
 from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
@@ -78,7 +77,7 @@ def response(resp):
 
         # torrent title
         page_a = result.xpath(xpath_title)[0]
-        title = escape(extract_text(page_a))
+        title = extract_text(page_a)
 
         # link to the page
         href = page_a.attrib.get('href')
@@ -90,7 +89,7 @@ def response(resp):
         try:
             file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')
             file_size = int(float(file_size) * get_filesize_mul(suffix))
-        except Exception as e:
+        except:
             file_size = None
 
         # seed count
@@ -105,7 +104,6 @@ def response(resp):
         # content string contains all information not included into template
         content = 'Category: "{category}". Downloaded {downloads} times.'
         content = content.format(category=category, downloads=downloads)
-        content = escape(content)
 
         results.append({'url': href,
                         'title': title,

searx/engines/openstreetmap.py

@@ -43,7 +43,7 @@ def response(resp):
         if 'display_name' not in r:
             continue
 
-        title = r['display_name']
+        title = r['display_name'] or u''
         osm_type = r.get('osm_type', r.get('type'))
         url = result_base_url.format(osm_type=osm_type,
                                      osm_id=r['osm_id'])

searx/engines/piratebay.py

@@ -9,7 +9,6 @@
 # @parse       url, title, content, seed, leech, magnetlink
 
 from urlparse import urljoin
-from cgi import escape
 from urllib import quote
 from lxml import html
 from operator import itemgetter
@@ -62,7 +61,7 @@ def response(resp):
         link = result.xpath('.//div[@class="detName"]//a')[0]
         href = urljoin(url, link.attrib.get('href'))
         title = extract_text(link)
-        content = escape(extract_text(result.xpath(content_xpath)))
+        content = extract_text(result.xpath(content_xpath))
         seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
 
         # convert seed to int if possible

searx/engines/reddit.py

@@ -11,7 +11,6 @@
 """
 
 import json
-from cgi import escape
 from urllib import urlencode
 from urlparse import urlparse, urljoin
 from datetime import datetime
@@ -68,7 +67,7 @@ def response(resp):
             img_results.append(params)
         else:
             created = datetime.fromtimestamp(data['created_utc'])
-            content = escape(data['selftext'])
+            content = data['selftext']
             if len(content) > 500:
                 content = content[:500] + '...'
             params['content'] = content

searx/engines/searchcode_doc.py

@@ -44,20 +44,12 @@ def response(resp):
     # parse results
     for result in search_results.get('results', []):
         href = result['url']
-        title = "[" + result['type'] + "] " +\
+        title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])
-                result['namespace'] +\
-                " " + result['name']
-        content = '<span class="highlight">[' +\
-                  result['type'] + "] " +\
-                  result['name'] + " " +\
-                  result['synopsis'] +\
-                  "</span><br />" +\
-                  result['description']
 
         # append result
         results.append({'url': href,
                         'title': title,
-                        'content': content})
+                        'content': result['description']})
 
     # return results
     return results

searx/engines/seedpeer.py

@@ -9,7 +9,6 @@
 # @parse       url, title, content, seed, leech, magnetlink
 
 from urlparse import urljoin
-from cgi import escape
 from urllib import quote
 from lxml import html
 from operator import itemgetter

searx/engines/spotify.py

@@ -46,10 +46,11 @@ def response(resp):
         if result['type'] == 'track':
             title = result['name']
             url = result['external_urls']['spotify']
-            content = result['artists'][0]['name'] +\
+            content = '{} - {} - {}'.format(
-                " • " +\
+                result['artists'][0]['name'],
-                result['album']['name'] +\
+                result['album']['name'],
-                " • " + result['name']
+                result['name'])
+
             embedded = embedded_url.format(audioid=result['id'])
 
             # append result

searx/engines/stackoverflow.py

@@ -11,7 +11,6 @@
 """
 
 from urlparse import urljoin
-from cgi import escape
 from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text
@@ -48,8 +47,8 @@ def response(resp):
     for result in dom.xpath(results_xpath):
         link = result.xpath(link_xpath)[0]
         href = urljoin(url, link.attrib.get('href'))
-        title = escape(extract_text(link))
+        title = extract_text(link)
-        content = escape(extract_text(result.xpath(content_xpath)))
+        content = extract_text(result.xpath(content_xpath))
 
         # append result
         results.append({'url': href,

searx/engines/startpage.py

@@ -11,7 +11,6 @@
 # @todo        paging
 
 from lxml import html
-from cgi import escape
 from dateutil import parser
 from datetime import datetime, timedelta
 import re
@@ -79,10 +78,10 @@ def response(resp):
         if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):
             continue
 
-        title = escape(extract_text(link))
+        title = extract_text(link)
 
         if result.xpath('./p[@class="desc clk"]'):
-            content = escape(extract_text(result.xpath('./p[@class="desc clk"]')))
+            content = extract_text(result.xpath('./p[@class="desc clk"]'))
         else:
             content = ''
 

searx/engines/subtitleseeker.py

@@ -10,7 +10,6 @@
  @parse       url, title, content
 """
 
-from cgi import escape
 from urllib import quote_plus
 from lxml import html
 from searx.languages import language_codes
@@ -59,7 +58,7 @@ def response(resp):
         elif search_lang:
             href = href + search_lang + '/'
 
-        title = escape(extract_text(link))
+        title = extract_text(link)
 
         content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
         content = content + " - "
@@ -75,7 +74,7 @@ def response(resp):
         # append result
         results.append({'url': href,
                         'title': title,
-                        'content': escape(content)})
+                        'content': content})
 
     # return results
     return results

searx/engines/swisscows.py

@@ -10,7 +10,6 @@
  @parse       url, title, content
 """
 
-from cgi import escape
 from json import loads
 from urllib import urlencode, unquote
 import re
@@ -78,7 +77,7 @@ def response(resp):
 
             # append result
             results.append({'url': result['SourceUrl'],
-                            'title': escape(result['Title']),
+                            'title': result['Title'],
                             'content': '',
                             'img_src': img_url,
                             'template': 'images.html'})
@@ -90,8 +89,8 @@ def response(resp):
 
             # append result
             results.append({'url': result_url,
-                            'title': escape(result_title),
+                            'title': result_title,
-                            'content': escape(result_content)})
+                            'content': result_content})
 
     # parse images
     for result in json.get('Images', []):
@@ -100,7 +99,7 @@ def response(resp):
 
         # append result
         results.append({'url': result['SourceUrl'],
-                        'title': escape(result['Title']),
+                        'title': result['Title'],
                         'content': '',
                         'img_src': img_url,
                         'template': 'images.html'})

searx/engines/tokyotoshokan.py

@@ -11,7 +11,6 @@
 """
 
 import re
-from cgi import escape
 from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text

searx/engines/torrentz.py

@@ -12,7 +12,6 @@
 """
 
 import re
-from cgi import escape
 from urllib import urlencode
 from lxml import html
 from searx.engines.xpath import extract_text

searx/engines/translated.py

@@ -9,7 +9,6 @@
  @parse       url, title, content
 """
 import re
-from cgi import escape
 from searx.utils import is_valid_lang
 
 categories = ['general']
@@ -52,14 +51,14 @@ def request(query, params):
 def response(resp):
     results = []
     results.append({
-        'url': escape(web_url.format(
+        'url': web_url.format(
             from_lang=resp.search_params['from_lang'][2],
             to_lang=resp.search_params['to_lang'][2],
-            query=resp.search_params['query'])),
+            query=resp.search_params['query']),
-        'title': escape('[{0}-{1}] {2}'.format(
+        'title': '[{0}-{1}] {2}'.format(
             resp.search_params['from_lang'][1],
             resp.search_params['to_lang'][1],
-            resp.search_params['query'])),
+            resp.search_params['query']),
-        'content': escape(resp.json()['responseData']['translatedText'])
+        'content': resp.json()['responseData']['translatedText']
     })
     return results

searx/engines/wolframalpha_noapi.py

@@ -8,7 +8,6 @@
 # @stable      no
 # @parse       url, infobox
 
-from cgi import escape
 from json import loads
 from time import time
 from urllib import urlencode

searx/engines/yandex.py

@@ -9,7 +9,6 @@
  @parse       url, title, content
 """
 
-from cgi import escape
 from urllib import urlencode
 from lxml import html
 from searx.search import logger
@@ -52,8 +51,8 @@ def response(resp):
     for result in dom.xpath(results_xpath):
         try:
             res = {'url': result.xpath(url_xpath)[0],
-                   'title': escape(''.join(result.xpath(title_xpath))),
+                   'title': ''.join(result.xpath(title_xpath)),
-                   'content': escape(''.join(result.xpath(content_xpath)))}
+                   'content': ''.join(result.xpath(content_xpath))}
         except:
             logger.exception('yandex parse crash')
             continue

searx/plugins/doai_rewrite.py

@@ -27,5 +27,5 @@ def on_result(request, search, result):
             if doi.endswith(suffix):
                 doi = doi[:-len(suffix)]
         result['url'] = 'http://doai.io/' + doi
-        result['parsed_url'] = urlparse(ctx['result']['url'])
+        result['parsed_url'] = urlparse(result['url'])
     return True

searx/preferences.py

@@ -49,28 +49,32 @@ class StringSetting(Setting):
 class EnumStringSetting(Setting):
     """Setting of a value which can only come from the given choices"""
 
+    def _validate_selection(self, selection):
+        if selection not in self.choices:
+            raise ValidationException('Invalid value: "{0}"'.format(selection))
+
     def _post_init(self):
         if not hasattr(self, 'choices'):
             raise MissingArgumentException('Missing argument: choices')
-
+        self._validate_selection(self.value)
-        if self.value != '' and self.value not in self.choices:
-            raise ValidationException('Invalid default value: {0}'.format(self.value))
 
     def parse(self, data):
-        if data not in self.choices and data != self.value:
+        self._validate_selection(data)
-            raise ValidationException('Invalid choice: {0}'.format(data))
         self.value = data
 
 
 class MultipleChoiceSetting(EnumStringSetting):
     """Setting of values which can only come from the given choices"""
 
+    def _validate_selections(self, selections):
+        for item in selections:
+            if item not in self.choices:
+                raise ValidationException('Invalid value: "{0}"'.format(selections))
+
     def _post_init(self):
         if not hasattr(self, 'choices'):
             raise MissingArgumentException('Missing argument: choices')
-        for item in self.value:
+        self._validate_selections(self.value)
-            if item not in self.choices:
-                raise ValidationException('Invalid default value: {0}'.format(self.value))
 
     def parse(self, data):
         if data == '':
@@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting):
             return
 
         elements = data.split(',')
-        for item in elements:
+        self._validate_selections(elements)
-            if item not in self.choices:
-                raise ValidationException('Invalid choice: {0}'.format(item))
         self.value = elements
 
     def parse_form(self, data):
@@ -214,11 +216,12 @@ class Preferences(object):
         super(Preferences, self).__init__()
 
         self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
-                                   'language': EnumStringSetting('all', choices=LANGUAGE_CODES),
+                                   'language': EnumStringSetting(settings['search']['language'],
+                                                                 choices=LANGUAGE_CODES),
                                    'locale': EnumStringSetting(settings['ui']['default_locale'],
-                                                               choices=settings['locales'].keys()),
+                                                               choices=settings['locales'].keys() + ['']),
                                    'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
-                                                                     choices=autocomplete.backends.keys()),
+                                                                     choices=autocomplete.backends.keys() + ['']),
                                    'image_proxy': MapSetting(settings['server']['image_proxy'],
                                                              map={'': settings['server']['image_proxy'],
                                                                   '0': False,

searx/results.py

@@ -146,16 +146,17 @@ class ResultContainer(object):
                 self._number_of_results.append(result['number_of_results'])
                 results.remove(result)
 
-        with RLock():
+        if engine_name in engines:
-            engines[engine_name].stats['search_count'] += 1
+            with RLock():
-            engines[engine_name].stats['result_count'] += len(results)
+                engines[engine_name].stats['search_count'] += 1
+                engines[engine_name].stats['result_count'] += len(results)
 
         if not results:
             return
 
         self.results[engine_name].extend(results)
 
-        if not self.paging and engines[engine_name].paging:
+        if not self.paging and engine_name in engines and engines[engine_name].paging:
             self.paging = True
 
         for i, result in enumerate(results):

searx/search.py

@@ -24,6 +24,7 @@ import searx.poolrequests as requests_lib
 from searx.engines import (
     categories, engines
 )
+from searx.answerers import ask
 from searx.utils import gen_useragent
 from searx.query import RawTextQuery, SearchQuery
 from searx.results import ResultContainer
@@ -300,6 +301,14 @@ class Search(object):
         # start time
         start_time = time()
 
+        # answeres ?
+        answerers_results = ask(self.search_query)
+
+        if answerers_results:
+            for results in answerers_results:
+                self.result_container.extend('answer', results)
+            return self.result_container
+
         # init vars
         requests = []
 

searx/settings.yml

@@ -5,6 +5,7 @@ general:
 search:
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
     autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
+    language : "all"
 
 server:
     port : 8888

searx/settings_robot.yml

@@ -5,6 +5,7 @@ general:
 search:
     safe_search : 0
     autocomplete : ""
+    language: "all"
 
 server:
     port : 11111

searx/static/plugins/js/infinite_scroll.js

@@ -5,7 +5,7 @@ $(document).ready(function() {
             var formData = $('#pagination form:last').serialize();
             if (formData) {
                 $('#pagination').html('<div class="loading-spinner"></div>');
-                $.post('/', formData, function (data) {
+                $.post('./', formData, function (data) {
                     var body = $(data);
                     $('#pagination').remove();
                     $('#main_results').append('<hr/>');

searx/templates/courgette/opensearch_response_rss.xml

@@ -3,14 +3,14 @@
      xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
      xmlns:atom="http://www.w3.org/2005/Atom">
   <channel>
-    <title>Searx search: {{ q }}</title>
+    <title>Searx search: {{ q|e }}</title>
-    <link>{{ base_url }}?q={{ q }}</link>
+    <link>{{ base_url }}?q={{ q|e }}</link>
-    <description>Search results for "{{ q }}" - searx</description>
+    <description>Search results for "{{ q|e }}" - searx</description>
     <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
     <opensearch:startIndex>1</opensearch:startIndex>
     <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
     <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
-    <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />
+    <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
     {% for r in results %}
     <item>
       <title>{{ r.title }}</title>

searx/templates/courgette/results.html

@@ -1,6 +1,6 @@
 {% extends "courgette/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
 {% block content %}
 <div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div>
 <div class="small search center">
@@ -17,7 +17,7 @@
             {% for output_type in ('csv', 'json', 'rss') %}
             <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
                 <div class="left">
-                    <input type="hidden" name="q" value="{{ q }}" />
+                    <input type="hidden" name="q" value="{{ q|e }}" />
                     <input type="hidden" name="format" value="{{ output_type }}" />
                     {% for category in selected_categories %}
                     <input type="hidden" name="category_{{ category }}" value="1"/>
@@ -62,7 +62,7 @@
         {% if pageno > 1 %}
             <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
                 <div class="left">
-                    <input type="hidden" name="q" value="{{ q }}" />
+                    <input type="hidden" name="q" value="{{ q|e }}" />
                     {% for category in selected_categories %}
                     <input type="hidden" name="category_{{ category }}" value="1"/>
                     {% endfor %}
@@ -76,7 +76,7 @@
                 {% for category in selected_categories %}
                 <input type="hidden" name="category_{{ category }}" value="1"/>
                 {% endfor %}
-                <input type="hidden" name="q" value="{{ q }}" />
+                <input type="hidden" name="q" value="{{ q|e }}" />
                 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
                 <input type="submit" value="{{ _('next page') }} >>" />
             </div>

searx/templates/legacy/opensearch_response_rss.xml

@@ -3,14 +3,14 @@
      xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
      xmlns:atom="http://www.w3.org/2005/Atom">
   <channel>
-    <title>Searx search: {{ q }}</title>
+    <title>Searx search: {{ q|e }}</title>
-    <link>{{ base_url }}?q={{ q }}</link>
+    <link>{{ base_url }}?q={{ q|e }}</link>
-    <description>Search results for "{{ q }}" - searx</description>
+    <description>Search results for "{{ q|e }}" - searx</description>
     <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
     <opensearch:startIndex>1</opensearch:startIndex>
     <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
     <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
-    <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />
+    <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
     {% for r in results %}
     <item>
       <title>{{ r.title }}</title>

searx/templates/legacy/results.html

@@ -1,6 +1,6 @@
 {% extends "legacy/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
 {% block content %}
 <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>
 <div class="small search center">
@@ -18,7 +18,7 @@
         {% for output_type in ('csv', 'json', 'rss') %}
         <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
             <div class="left">
-            <input type="hidden" name="q" value="{{ q }}" />
+            <input type="hidden" name="q" value="{{ q|e }}" />
             <input type="hidden" name="format" value="{{ output_type }}" />
             {% for category in selected_categories %}
             <input type="hidden" name="category_{{ category }}" value="1"/>
@@ -73,7 +73,7 @@
         {% if pageno > 1 %}
             <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">
                 <div class="{% if rtl %}right{% else %}left{% endif %}">
-                <input type="hidden" name="q" value="{{ q }}" />
+                <input type="hidden" name="q" value="{{ q|e }}" />
                 {% for category in selected_categories %}
                 <input type="hidden" name="category_{{ category }}" value="1"/>
                 {% endfor %}
@@ -87,7 +87,7 @@
                 {% for category in selected_categories %}
                 <input type="hidden" name="category_{{ category }}" value="1"/>
                 {% endfor %}
-                <input type="hidden" name="q" value="{{ q }}" />
+                <input type="hidden" name="q" value="{{ q|e }}" />
                 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
                 <input type="submit" value="{{ _('next page') }} >>" />
             </div>

searx/templates/oscar/base.html

@@ -1,3 +1,4 @@
+{% from 'oscar/macros.html' import icon %}
 <!DOCTYPE html>
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}>
 <head>
@@ -54,6 +55,20 @@
 <body>
     {% include 'oscar/navbar.html' %}
     <div class="container">
+    {% if errors %}
+        <div class="alert alert-danger fade in" role="alert">
+            <button class="close" data-dismiss="alert" type="button">
+                <span aria-hidden="true">×</span>
+                <span class="sr-only">{{ _('Close') }}</span>
+            </button>
+            <strong class="lead">{{ icon('info-sign') }} {{ _('Error!') }}</strong>
+            <ul>
+            {% for message in errors %}
+                <li>{{ message }}</li>
+            {% endfor %}
+            </ul>
+        </div>
+    {% endif %}
 
     {% block site_alert_error %}
     {% endblock %}

searx/templates/oscar/opensearch_response_rss.xml

@@ -3,14 +3,14 @@
      xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"
      xmlns:atom="http://www.w3.org/2005/Atom">
   <channel>
-    <title>Searx search: {{ q }}</title>
+    <title>Searx search: {{ q|e }}</title>
-    <link>{{ base_url }}?q={{ q }}</link>
+    <link>{{ base_url }}?q={{ q|e }}</link>
-    <description>Search results for "{{ q }}" - searx</description>
+    <description>Search results for "{{ q|e }}" - searx</description>
     <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>
     <opensearch:startIndex>1</opensearch:startIndex>
     <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>
     <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>
-    <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />
+    <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />
     {% for r in results %}
     <item>
       <title>{{ r.title }}</title>

searx/templates/oscar/preferences.html

@@ -12,6 +12,7 @@
           <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
           <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
           <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
+          {% if answerers %}<li><a href="#tab_answerers" role="tab" data-toggle="tab">{{ _('Answerers') }}</a></li>{% endif %}
           <li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li>
         </ul>
 
@@ -224,6 +225,34 @@
                 </fieldset>
             </div>
 
+            {% if answerers %}
+            <div class="tab-pane active_if_nojs" id="tab_answerers">
+                <noscript>
+                    <h3>{{ _('Answerers') }}</h3>
+                </noscript>
+                <p class="text-muted" style="margin:20px 0;">
+                    {{ _('This is the list of searx\'s instant answering modules.') }}
+                </p>
+                <table class="table table-striped">
+                    <tr>
+                        <th class="text-muted">{{ _('Name') }}</th>
+                        <th class="text-muted">{{ _('Keywords') }}</th>
+                        <th class="text-muted">{{ _('Description') }}</th>
+                        <th class="text-muted">{{ _('Examples') }}</th>
+                    </tr>
+
+                    {% for answerer in answerers %}
+                    <tr>
+                        <td class="text-muted">{{ answerer.info.name }}</td>
+                        <td class="text-muted">{{ answerer.keywords|join(', ') }}</td>
+                        <td class="text-muted">{{ answerer.info.description }}</td>
+                        <td class="text-muted">{{ answerer.info.examples|join(', ') }}</td>
+                    </tr>
+                    {% endfor %}
+                </table>
+            </div>
+            {% endif %}
+
             <div class="tab-pane active_if_nojs" id="tab_cookies">
                 <noscript>
                     <h3>{{ _('Cookies') }}</h3>

searx/templates/oscar/result_templates/images.html

@@ -13,7 +13,12 @@
             </div>
             <div class="modal-body">
                 <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
-                {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}
+                {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
+                {% if result.content %}
+                    <p class="result-content">
+                        {{ result.content }}
+                    </p>
+                {% endif %}
             </div>
             <div class="modal-footer">
                 <div class="clearfix"></div>

searx/templates/oscar/results.html

@@ -1,6 +1,6 @@
 {% extends "oscar/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}&amp;time_range={{ time_range }}">{% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}&amp;time_range={{ time_range }}">{% endblock %}
 {% block content %}
     <div class="row">
         <div class="col-sm-8" id="main_results">
@@ -37,9 +37,9 @@
             <div id="pagination">
                 <div class="pull-left">
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
-                        <input type="hidden" name="q" value="{{ q }}" />
+                        <input type="hidden" name="q" value="{{ q|e }}" />
                         {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
-                        <input type="hidden" name="q" value="{{ q }}" />
+                        <input type="hidden" name="q" value="{{ q|e }}" />
                         <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
                         <input type="hidden" name="time_range" value="{{ time_range }}" />
                         <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>
@@ -59,7 +59,7 @@
             <div id="pagination">
                 <div class="pull-left">
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
-                        <input type="hidden" name="q" value="{{ q }}" />
+                        <input type="hidden" name="q" value="{{ q|e }}" />
                         {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
                         <input type="hidden" name="pageno" value="{{ pageno-1 }}" />
                         <input type="hidden" name="time_range" value="{{ time_range }}" />
@@ -69,7 +69,7 @@
                 <div class="pull-right">
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"  class="pull-left">
                         {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
-                        <input type="hidden" name="q" value="{{ q }}" />
+                        <input type="hidden" name="q" value="{{ q|e }}" />
                         <input type="hidden" name="pageno" value="{{ pageno+1 }}" />
                         <input type="hidden" name="time_range" value="{{ time_range }}" />
                         <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>
@@ -130,7 +130,7 @@
                     <div class="clearfix"></div>
                     {% for output_type in ('csv', 'json', 'rss') %}
                     <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
-                        <input type="hidden" name="q" value="{{ q }}">
+                        <input type="hidden" name="q" value="{{ q|e }}">
                         <input type="hidden" name="format" value="{{ output_type }}">
                         {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}
                         <input type="hidden" name="pageno" value="{{ pageno }}">

searx/templates/pix-art/results.html

@@ -5,7 +5,7 @@
     {% endfor %}
 {% else %}
 {% extends "pix-art/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
 {% block meta %}{% endblock %}
 {% block content %}
 <div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div>
@@ -25,8 +25,8 @@
     </span>
     <div id="pagination">
         <br />
-        <input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />
+        <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />
     </div>
 </div>
 {% endblock %}
-{% endif %}
+{% endif %}

searx/utils.py

@@ -6,7 +6,10 @@ import re
 from babel.dates import format_date
 from codecs import getincrementalencoder
 from HTMLParser import HTMLParser
+from imp import load_source
+from os.path import splitext, join
 from random import choice
+import sys
 
 from searx.version import VERSION_STRING
 from searx.languages import language_codes
@@ -285,3 +288,13 @@ def is_valid_lang(lang):
             if l[1].lower() == lang.lower():
                 return (True, l[0][:2], l[1].lower())
         return False
+
+
+def load_module(filename, module_dir):
+    modname = splitext(filename)[0]
+    if modname in sys.modules:
+        del sys.modules[modname]
+    filepath = join(module_dir, filename)
+    module = load_source(modname, filepath)
+    module.name = modname
+    return module

searx/webapp.py

@@ -40,7 +40,7 @@ except:
     logger.critical("cannot import dependency: pygments")
     from sys import exit
     exit(1)
-
+from cgi import escape
 from datetime import datetime, timedelta
 from urllib import urlencode
 from urlparse import urlparse, urljoin
@@ -62,11 +62,12 @@ from searx.utils import (
 )
 from searx.version import VERSION_STRING
 from searx.languages import language_codes
-from searx.search import Search, SearchWithPlugins, get_search_query_from_webapp
+from searx.search import SearchWithPlugins, get_search_query_from_webapp
-from searx.query import RawTextQuery, SearchQuery
+from searx.query import RawTextQuery
 from searx.autocomplete import searx_bang, backends as autocomplete_backends
 from searx.plugins import plugins
 from searx.preferences import Preferences, ValidationException
+from searx.answerers import answerers
 
 # check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.
 # They are needed for SSL connection without trouble, see #298
@@ -344,6 +345,8 @@ def render(template_name, override_theme=None, **kwargs):
 
     kwargs['cookies'] = request.cookies
 
+    kwargs['errors'] = request.errors
+
     kwargs['instance_name'] = settings['general']['instance_name']
 
     kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
@@ -364,15 +367,16 @@ def render(template_name, override_theme=None, **kwargs):
 
 @app.before_request
 def pre_request():
-    # merge GET, POST vars
+    request.errors = []
+
     preferences = Preferences(themes, categories.keys(), engines, plugins)
+    request.preferences = preferences
     try:
         preferences.parse_cookies(request.cookies)
     except:
-        # TODO throw error message to the user
+        request.errors.append(gettext('Invalid settings, please edit your preferences'))
-        logger.warning('Invalid config')
-    request.preferences = preferences
 
+    # merge GET, POST vars
     # request.form
     request.form = dict(request.form.items())
     for k, v in request.args.items():
@@ -397,7 +401,7 @@ def index():
     Supported outputs: html, json, csv, rss.
     """
 
-    if not request.args and not request.form:
+    if request.form.get('q') is None:
         return render(
             'index.html',
         )
@@ -411,6 +415,8 @@ def index():
         search = SearchWithPlugins(search_query, request)
         result_container = search.search()
     except:
+        request.errors.append(gettext('search error'))
+        logger.exception('search error')
         return render(
             'index.html',
         )
@@ -427,8 +433,10 @@ def index():
     for result in results:
         if output_format == 'html':
             if 'content' in result and result['content']:
-                result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8'))
+                result['content'] = highlight_content(escape(result['content'][:1024]),
-            result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8'))
+                                                      search_query.query.encode('utf-8'))
+            result['title'] = highlight_content(escape(result['title'] or u''),
+                                                search_query.query.encode('utf-8'))
         else:
             if result.get('content'):
                 result['content'] = html_to_text(result['content']).strip()
@@ -572,7 +580,7 @@ def preferences():
         try:
             request.preferences.parse_form(request.form)
         except ValidationException:
-            # TODO use flash feature of flask
+            request.errors.append(gettext('Invalid settings, please edit your preferences'))
             return resp
         return request.preferences.save(resp)
 
@@ -609,6 +617,7 @@ def preferences():
                   language_codes=language_codes,
                   engines_by_category=categories,
                   stats=stats,
+                  answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
                   disabled_engines=disabled_engines,
                   autocomplete_backends=autocomplete_backends,
                   shortcuts={y: x for x, y in engine_shortcuts.items()},

tests/unit/engines/test_bing.py

@@ -14,14 +14,12 @@ class TestBingEngine(SearxTestCase):
         params = bing.request(query, dicto)
         self.assertTrue('url' in params)
         self.assertTrue(query in params['url'])
+        self.assertTrue('language%3AFR' in params['url'])
         self.assertTrue('bing.com' in params['url'])
-        self.assertTrue('SRCHHPGUSR' in params['cookies'])
-        self.assertTrue('fr' in params['cookies']['SRCHHPGUSR'])
 
         dicto['language'] = 'all'
         params = bing.request(query, dicto)
-        self.assertTrue('SRCHHPGUSR' in params['cookies'])
+        self.assertTrue('language' not in params['url'])
-        self.assertTrue('en' in params['cookies']['SRCHHPGUSR'])
 
     def test_response(self):
         self.assertRaises(AttributeError, bing.response, None)

tests/unit/engines/test_deezer.py

@@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], 'Title of track')
         self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042')
-        self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')
+        self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
         self.assertTrue('100' in results[0]['embedded'])
 
         json = r"""

tests/unit/engines/test_flickr.py

@@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase):
         self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
         self.assertTrue('o.jpg' in results[0]['img_src'])
         self.assertTrue('n.jpg' in results[0]['thumbnail_src'])
-        self.assertTrue('Owner' in results[0]['content'])
+        self.assertTrue('Owner' in results[0]['author'])
         self.assertTrue('Description' in results[0]['content'])
 
         json = r"""
@@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase):
         self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
         self.assertTrue('z.jpg' in results[0]['img_src'])
         self.assertTrue('z.jpg' in results[0]['thumbnail_src'])
-        self.assertTrue('Owner' in results[0]['content'])
+        self.assertTrue('Owner' in results[0]['author'])
         self.assertTrue('Description' in results[0]['content'])
 
         json = r"""
@@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase):
         self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')
         self.assertTrue('o.jpg' in results[0]['img_src'])
         self.assertTrue('o.jpg' in results[0]['thumbnail_src'])
-        self.assertTrue('Owner' in results[0]['content'])
+        self.assertTrue('Owner' in results[0]['author'])
         self.assertTrue('Description' in results[0]['content'])
 
         json = r"""

tests/unit/engines/test_flickr_noapi.py

@@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
         self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
         self.assertIn('k.jpg', results[0]['img_src'])
         self.assertIn('n.jpg', results[0]['thumbnail_src'])
-        self.assertIn('Owner', results[0]['content'])
+        self.assertIn('Owner', results[0]['author'])
 
         # no n size, only the z size
         json = """
@@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
         self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
         self.assertIn('z.jpg', results[0]['img_src'])
         self.assertIn('z.jpg', results[0]['thumbnail_src'])
-        self.assertIn('Owner', results[0]['content'])
+        self.assertIn('Owner', results[0]['author'])
 
         # no z or n size
         json = """
@@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
         self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')
         self.assertIn('o.jpg', results[0]['img_src'])
         self.assertIn('o.jpg', results[0]['thumbnail_src'])
-        self.assertIn('Owner', results[0]['content'])
+        self.assertIn('Owner', results[0]['author'])
 
         # no image test
         json = """

tests/unit/engines/test_kickass.py

@@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], 'This should be the title')
         self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
-        self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
+        self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
         self.assertEqual(results[0]['filesize'], 449)
@@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(len(results), 5)
         self.assertEqual(results[0]['title'], 'This should be the title')
         self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
-        self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
+        self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
         self.assertEqual(results[0]['files'], 4)

tests/unit/engines/test_searchcode_doc.py

@@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], '[Type] Namespace test')
         self.assertEqual(results[0]['url'], 'http://url')
-        self.assertIn('Synopsis', results[0]['content'])
-        self.assertIn('Type', results[0]['content'])
-        self.assertIn('test', results[0]['content'])
         self.assertIn('Description', results[0]['content'])
 
         json = r"""

tests/unit/engines/test_spotify.py

@@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], 'Title of track')
         self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa')
-        self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')
+        self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')
         self.assertIn('1000', results[0]['embedded'])
 
         json = """

tests/unit/test_answerers.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+from mock import Mock
+
+from searx.answerers import answerers
+from searx.testing import SearxTestCase
+
+
+class AnswererTest(SearxTestCase):
+
+    def test_unicode_input(self):
+        query = Mock()
+        unicode_payload = u'árvíztűrő tükörfúrógép'
+        for answerer in answerers:
+            query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload)
+            self.assertTrue(isinstance(answerer.answer(query), list))

tests/unit/test_webapp.py

@@ -5,6 +5,7 @@ from mock import Mock
 from urlparse import ParseResult
 from searx import webapp
 from searx.testing import SearxTestCase
+from searx.search import Search
 
 
 class ViewsTestCase(SearxTestCase):
@@ -41,7 +42,7 @@ class ViewsTestCase(SearxTestCase):
                                                 results_number=lambda: 3,
                                                 results_length=lambda: len(self.test_results))
 
-        webapp.Search.search = search_mock
+        Search.search = search_mock
 
         def get_current_theme_name_mock(override=None):
             return 'legacy'