new preferences handling

Preferences class was introduced in order to handle user preferences. Right now
it parses cookies and the form in preferences. Also it can retrieve settings
based on the name of the setting.

ATTENTION
Please note that engine preferences are handled differently from now on. So it
introduces incompatible changes. Every user who has saved preferences should reset and
save his/her settings again.

This change was needed, because everytime a default disabled engine was
added saved user preferences would broke. Now engine setting tracking is
fixed.
This commit is contained in:
Noemi Vanyi 2016-04-08 16:38:05 +02:00
parent 9331fc28a8
commit fe691a0988
6 changed files with 315 additions and 167 deletions

269
searx/preferences.py Normal file
View File

@ -0,0 +1,269 @@
from searx import settings, autocomplete
from searx.languages import language_codes as languages
COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years
LANGUAGE_CODES = [l[0] for l in languages]
LANGUAGE_CODES.append('all')
DISABLED = 0
ENABLED = 1
class MissingArgumentException(Exception):
pass
class ValidationException(Exception):
pass
class Setting(object):
"""Base class of user settings"""
def __init__(self, default_value, **kwargs):
super(Setting, self).__init__()
self.value = default_value
for key, value in kwargs.iteritems():
setattr(self, key, value)
self._post_init()
def _post_init(self):
pass
def parse(self, data):
self.value = data
def get_value(self):
return self.value
def save(self, name, resp):
resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE)
class StringSetting(Setting):
"""Setting of plain string values"""
pass
class EnumStringSetting(Setting):
"""Setting of a value which can only come from the given choices"""
def _post_init(self):
if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices')
if self.value != '' and self.value not in self.choices:
raise ValidationException('Invalid default value: {0}'.format(self.value))
def parse(self, data):
if data not in self.choices and data != self.value:
raise ValidationException('Invalid choice: {0}'.format(data))
self.value = data
class MultipleChoiceSetting(EnumStringSetting):
"""Setting of values which can only come from the given choices"""
def _post_init(self):
if not hasattr(self, 'choices'):
raise MissingArgumentException('Missing argument: choices')
for item in self.value:
if item not in self.choices:
raise ValidationException('Invalid default value: {0}'.format(self.value))
def parse(self, data):
if data == '':
self.value = []
return
elements = data.split(',')
for item in elements:
if item not in self.choices:
raise ValidationException('Invalid choice: {0}'.format(item))
self.value = elements
def parse_form(self, data):
self.value = []
for choice in data:
if choice in self.choices and choice not in self.value:
self.value.append(choice)
def save(self, name, resp):
resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)
class MapSetting(Setting):
"""Setting of a value that has to be translated in order to be storable"""
def _post_init(self):
if not hasattr(self, 'map'):
raise MissingArgumentException('missing argument: map')
if self.value not in self.map.values():
raise ValidationException('Invalid default value')
def parse(self, data):
if data not in self.map:
raise ValidationException('Invalid choice: {0}'.format(data))
self.value = self.map[data]
self.key = data
def save(self, name, resp):
resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE)
class SwitchableSetting(Setting):
""" Base class for settings that can be turned on && off"""
def _post_init(self):
self.disabled = set()
self.enabled = set()
if not hasattr(self, 'choices'):
raise MissingArgumentException('missing argument: choices')
def transform_form_items(self, items):
return items
def transform_values(self, values):
return values
def parse_cookie(self, data):
if data[DISABLED] != '':
self.disabled = set(data[DISABLED].split(','))
if data[ENABLED] != '':
self.enabled = set(data[ENABLED].split(','))
def parse_form(self, items):
items = self.transform_form_items(items)
self.disabled = set()
self.enabled = set()
for choice in self.choices:
if choice['default_on']:
if choice['id'] in items:
self.disabled.add(choice['id'])
else:
if choice['id'] not in items:
self.enabled.add(choice['id'])
def save(self, resp):
resp.set_cookie('disabled_{0}'.format(self.value), ','.join(self.disabled), max_age=COOKIE_MAX_AGE)
resp.set_cookie('enabled_{0}'.format(self.value), ','.join(self.enabled), max_age=COOKIE_MAX_AGE)
def get_disabled(self):
disabled = self.disabled
for choice in self.choices:
if not choice['default_on'] and choice['id'] not in self.enabled:
disabled.add(choice['id'])
return self.transform_values(disabled)
def get_enabled(self):
enabled = self.enabled
for choice in self.choices:
if choice['default_on'] and choice['id'] not in self.disabled:
enabled.add(choice['id'])
return self.transform_values(enabled)
class EnginesSetting(SwitchableSetting):
def _post_init(self):
super(EnginesSetting, self)._post_init()
transformed_choices = []
for engine_name, engine in self.choices.iteritems():
for category in engine.categories:
transformed_choice = dict()
transformed_choice['default_on'] = not engine.disabled
transformed_choice['id'] = '{}__{}'.format(engine_name, category)
transformed_choices.append(transformed_choice)
self.choices = transformed_choices
def transform_form_items(self, items):
return [item[len('engine_'):].replace('_', ' ').replace(' ', '__') for item in items]
def transform_values(self, values):
if len(values) == 1 and values[0] == '':
return list()
transformed_values = []
for value in values:
engine, category = value.split('__')
transformed_values.append((engine, category))
return transformed_values
class PluginsSetting(SwitchableSetting):
def _post_init(self):
super(PluginsSetting, self)._post_init()
transformed_choices = []
for plugin in self.choices:
transformed_choice = dict()
transformed_choice['default_on'] = plugin.default_on
transformed_choice['id'] = plugin.id
transformed_choices.append(transformed_choice)
self.choices = transformed_choices
def transform_form_items(self, items):
return [item[len('plugin_'):] for item in items]
class Preferences(object):
"""Stores, validates and saves preferences to cookies"""
def __init__(self, themes, categories, engines, plugins):
super(Preferences, self).__init__()
self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
'language': EnumStringSetting('all', choices=LANGUAGE_CODES),
'locale': EnumStringSetting(settings['ui']['default_locale'],
choices=settings['locales'].keys()),
'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
choices=autocomplete.backends.keys()),
'image_proxy': MapSetting(settings['server']['image_proxy'],
map={'': settings['server']['image_proxy'],
'0': False,
'1': True}),
'method': EnumStringSetting('POST', choices=('GET', 'POST')),
'safesearch': MapSetting(settings['search']['safe_search'], map={'0': 0,
'1': 1,
'2': 2}),
'theme': EnumStringSetting(settings['ui']['default_theme'], choices=themes)}
self.engines = EnginesSetting('engines', choices=engines)
self.plugins = PluginsSetting('plugins', choices=plugins)
def parse_cookies(self, input_data):
for user_setting_name, user_setting in input_data.iteritems():
if user_setting_name in self.key_value_settings:
self.key_value_settings[user_setting_name].parse(user_setting)
elif user_setting_name == 'disabled_engines':
self.engines.parse_cookie([input_data['disabled_engines'], input_data['enabled_engines']])
elif user_setting_name == 'disabled_plugins':
self.plugins.parse_cookie([input_data['disabled_plugins'], input_data['enabled_plugins']])
def parse_form(self, input_data):
disabled_engines = []
enabled_categories = []
disabled_plugins = []
for user_setting_name, user_setting in input_data.iteritems():
if user_setting_name in self.key_value_settings:
self.key_value_settings[user_setting_name].parse(user_setting)
elif user_setting_name.startswith('engine_'):
disabled_engines.append(user_setting_name)
elif user_setting_name.startswith('category_'):
enabled_categories.append(user_setting_name[len('category_'):])
elif user_setting_name.startswith('plugin_'):
disabled_plugins.append(user_setting_name)
self.key_value_settings['categories'].parse_form(enabled_categories)
self.engines.parse_form(disabled_engines)
self.plugins.parse_form(disabled_plugins)
# cannot be used in case of engines or plugins
def get_value(self, user_setting_name):
if user_setting_name in self.key_value_settings:
return self.key_value_settings[user_setting_name].get_value()
def save(self, resp):
for user_setting_name, user_setting in self.key_value_settings.iteritems():
user_setting.save(user_setting_name, resp)
self.engines.save(resp)
self.plugins.save(resp)
return resp

View File

@ -23,7 +23,7 @@ from searx.engines import (
categories, engines
)
from searx.languages import language_codes
from searx.utils import gen_useragent, get_blocked_engines
from searx.utils import gen_useragent
from searx.query import Query
from searx.results import ResultContainer
from searx import logger
@ -140,15 +140,13 @@ class Search(object):
self.lang = 'all'
# set blocked engines
self.blocked_engines = get_blocked_engines(engines, request.cookies)
self.blocked_engines = request.preferences.engines.get_disabled()
self.result_container = ResultContainer()
self.request_data = {}
# set specific language if set
if request.cookies.get('language')\
and request.cookies['language'] in (x[0] for x in language_codes):
self.lang = request.cookies['language']
self.lang = request.preferences.get_value('language')
# set request method
if request.method == 'POST':
@ -294,11 +292,8 @@ class Search(object):
else:
request_params['language'] = self.lang
try:
# 0 = None, 1 = Moderate, 2 = Strict
request_params['safesearch'] = int(request.cookies.get('safesearch'))
except Exception:
request_params['safesearch'] = settings['search']['safe_search']
# 0 = None, 1 = Moderate, 2 = Strict
request_params['safesearch'] = request.preferences.get_value('safesearch')
# update request parameters dependent on
# search-engine (contained in engines folder)

View File

@ -4,7 +4,7 @@ general:
search:
safe_search : 0
autocomplete : 0
autocomplete : ""
server:
port : 11111

View File

@ -230,26 +230,3 @@ def list_get(a_list, index, default=None):
return a_list[index]
else:
return default
def get_blocked_engines(engines, cookies):
if 'blocked_engines' not in cookies:
return [(engine_name, category) for engine_name in engines
for category in engines[engine_name].categories if engines[engine_name].disabled]
blocked_engine_strings = cookies.get('blocked_engines', '').split(',')
blocked_engines = []
if not blocked_engine_strings:
return blocked_engines
for engine_string in blocked_engine_strings:
if engine_string.find('__') > -1:
engine, category = engine_string.split('__', 1)
if engine in engines and category in engines[engine].categories:
blocked_engines.append((engine, category))
elif engine_string in engines:
for category in engines[engine_string].categories:
blocked_engines.append((engine_string, category))
return blocked_engines

View File

@ -56,7 +56,7 @@ from searx.engines import (
from searx.utils import (
UnicodeWriter, highlight_content, html_to_text, get_themes,
get_static_files, get_result_templates, gen_useragent, dict_subset,
prettify_url, get_blocked_engines
prettify_url
)
from searx.version import VERSION_STRING
from searx.languages import language_codes
@ -64,6 +64,7 @@ from searx.search import Search
from searx.query import Query
from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx.plugins import plugins
from searx.preferences import Preferences
# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.
# They are needed for SSL connection without trouble, see #298
@ -109,8 +110,6 @@ for indice, theme in enumerate(themes):
for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
global_favicons[indice].extend(filenames)
cookie_max_age = 60 * 60 * 24 * 365 * 5 # 5 years
_category_names = (gettext('files'),
gettext('general'),
gettext('music'),
@ -222,9 +221,7 @@ def get_current_theme_name(override=None):
if override and override in themes:
return override
theme_name = request.args.get('theme',
request.cookies.get('theme',
default_theme))
theme_name = request.args.get('theme', request.preferences.get_value('theme'))
if theme_name not in themes:
theme_name = default_theme
return theme_name
@ -262,12 +259,8 @@ def image_proxify(url):
def render(template_name, override_theme=None, **kwargs):
blocked_engines = get_blocked_engines(engines, request.cookies)
autocomplete = request.cookies.get('autocomplete', settings['search']['autocomplete'])
if autocomplete not in autocomplete_backends:
autocomplete = None
blocked_engines = request.preferences.engines.get_disabled()
autocomplete = request.preferences.get_value('autocomplete')
nonblocked_categories = set(category for engine_name in engines
for category in engines[engine_name].categories
@ -295,7 +288,7 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['selected_categories'].append(c)
if not kwargs['selected_categories']:
cookie_categories = request.cookies.get('categories', '').split(',')
cookie_categories = request.preferences.get_value('categories')
for ccateg in cookie_categories:
if ccateg in categories:
kwargs['selected_categories'].append(ccateg)
@ -311,9 +304,9 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['searx_version'] = VERSION_STRING
kwargs['method'] = request.cookies.get('method', 'POST')
kwargs['method'] = request.preferences.get_value('method')
kwargs['safesearch'] = request.cookies.get('safesearch', str(settings['search']['safe_search']))
kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
# override url_for function in templates
kwargs['url_for'] = url_for_theme
@ -347,14 +340,18 @@ def render(template_name, override_theme=None, **kwargs):
@app.before_request
def pre_request():
# merge GET, POST vars
preferences = Preferences(themes, categories.keys(), engines, plugins)
preferences.parse_cookies(request.cookies)
request.preferences = preferences
request.form = dict(request.form.items())
for k, v in request.args.items():
if k not in request.form:
request.form[k] = v
request.user_plugins = []
allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')
disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')
allowed_plugins = preferences.plugins.get_enabled()
disabled_plugins = preferences.plugins.get_disabled()
for plugin in plugins:
if ((plugin.default_on and plugin.id not in disabled_plugins)
or plugin.id in allowed_plugins):
@ -486,7 +483,7 @@ def autocompleter():
request_data = request.args
# set blocked engines
blocked_engines = get_blocked_engines(engines, request.cookies)
blocked_engines = request.preferences.engines.get_disabled()
# parse query
query = Query(request_data.get('q', '').encode('utf-8'), blocked_engines)
@ -496,8 +493,8 @@ def autocompleter():
if not query.getSearchQuery():
return '', 400
# get autocompleter
completer = autocomplete_backends.get(request.cookies.get('autocomplete', settings['search']['autocomplete']))
# run autocompleter
completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
# parse searx specific autocompleter results like !bang
raw_results = searx_bang(query)
@ -532,117 +529,23 @@ def autocompleter():
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
"""Render preferences page.
"""Render preferences page && save user preferences"""
Settings that are going to be saved as cookies."""
lang = None
image_proxy = request.cookies.get('image_proxy', settings['server'].get('image_proxy'))
# save preferences
if request.method == 'POST':
resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
try:
request.preferences.parse_form(request.form)
except ValidationException:
# TODO use flash feature of flask
return resp
return request.preferences.save(resp)
if request.cookies.get('language')\
and request.cookies['language'] in (x[0] for x in language_codes):
lang = request.cookies['language']
blocked_engines = []
resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
if request.method == 'GET':
blocked_engines = get_blocked_engines(engines, request.cookies)
else: # on save
selected_categories = []
post_disabled_plugins = []
locale = None
autocomplete = ''
method = 'POST'
safesearch = settings['search']['safe_search']
for pd_name, pd in request.form.items():
if pd_name.startswith('category_'):
category = pd_name[9:]
if category not in categories:
continue
selected_categories.append(category)
elif pd_name == 'locale' and pd in settings['locales']:
locale = pd
elif pd_name == 'image_proxy':
image_proxy = pd
elif pd_name == 'autocomplete':
autocomplete = pd
elif pd_name == 'language' and (pd == 'all' or
pd in (x[0] for
x in language_codes)):
lang = pd
elif pd_name == 'method':
method = pd
elif pd_name == 'safesearch':
safesearch = pd
elif pd_name.startswith('engine_'):
if pd_name.find('__') > -1:
# TODO fix underscore vs space
engine_name, category = [x.replace('_', ' ') for x in
pd_name.replace('engine_', '', 1).split('__', 1)]
if engine_name in engines and category in engines[engine_name].categories:
blocked_engines.append((engine_name, category))
elif pd_name == 'theme':
theme = pd if pd in themes else default_theme
elif pd_name.startswith('plugin_'):
plugin_id = pd_name.replace('plugin_', '', 1)
if not any(plugin.id == plugin_id for plugin in plugins):
continue
post_disabled_plugins.append(plugin_id)
else:
resp.set_cookie(pd_name, pd, max_age=cookie_max_age)
disabled_plugins = []
allowed_plugins = []
for plugin in plugins:
if plugin.default_on:
if plugin.id in post_disabled_plugins:
disabled_plugins.append(plugin.id)
elif plugin.id not in post_disabled_plugins:
allowed_plugins.append(plugin.id)
resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)
resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)
resp.set_cookie(
'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),
max_age=cookie_max_age
)
if locale:
resp.set_cookie(
'locale', locale,
max_age=cookie_max_age
)
if lang:
resp.set_cookie(
'language', lang,
max_age=cookie_max_age
)
if selected_categories:
# cookie max age: 4 weeks
resp.set_cookie(
'categories', ','.join(selected_categories),
max_age=cookie_max_age
)
resp.set_cookie(
'autocomplete', autocomplete,
max_age=cookie_max_age
)
resp.set_cookie('method', method, max_age=cookie_max_age)
resp.set_cookie('safesearch', str(safesearch), max_age=cookie_max_age)
resp.set_cookie('image_proxy', image_proxy, max_age=cookie_max_age)
resp.set_cookie('theme', theme, max_age=cookie_max_age)
return resp
# render preferences
image_proxy = request.preferences.get_value('image_proxy')
lang = request.preferences.get_value('language')
blocked_engines = request.preferences.engines.get_disabled()
allowed_plugins = request.preferences.plugins.get_enabled()
# stats for preferences page
stats = {}
@ -664,7 +567,7 @@ def preferences():
return render('preferences.html',
locales=settings['locales'],
current_locale=get_locale(),
current_language=lang or 'all',
current_language=lang,
image_proxy=image_proxy,
language_codes=language_codes,
engines_by_category=categories,
@ -674,7 +577,7 @@ def preferences():
shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes,
plugins=plugins,
allowed_plugins=[plugin.id for plugin in request.user_plugins],
allowed_plugins=allowed_plugins,
theme=get_current_theme_name())
@ -750,7 +653,7 @@ Disallow: /preferences
def opensearch():
method = 'post'
if request.cookies.get('method', 'POST') == 'GET':
if request.preferences.get_value('method') == 'GET':
method = 'get'
# chrome/chromium only supports HTTP GET....

View File

@ -12,7 +12,6 @@ class ViewsTestCase(SearxTestCase):
def setUp(self):
webapp.app.config['TESTING'] = True # to get better error messages
self.app = webapp.app.test_client()
webapp.default_theme = 'default'
# set some defaults
self.test_results = [
@ -43,6 +42,11 @@ class ViewsTestCase(SearxTestCase):
webapp.Search.search = search_mock
def get_current_theme_name_mock(override=None):
return 'default'
webapp.get_current_theme_name = get_current_theme_name_mock
self.maxDiff = None # to see full diffs
def test_index_empty(self):