LibreMiami
/
libremiami-search
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'plugins'

This commit is contained in:
Adam Tauber 2015-03-15 11:38:07 +01:00
parent 4cc79533e7 f57149f912
commit bd92b43449
10 changed files with 238 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
searx/plugins/__init__.py Normal file
View File

@ -0,0 +1,48 @@
from searx.plugins import self_ip
from searx import logger
from sys import exit
logger = logger.getChild('plugins')
required_attrs = (('name', str),
('description', str),
('default_on', bool))
class Plugin():
default_on = False
name = 'Default plugin'
description = 'Default plugin description'
class PluginStore():
def __init__(self):
self.plugins = []
def __iter__(self):
for plugin in self.plugins:
yield plugin
def register(self, *plugins):
for plugin in plugins:
for plugin_attr, plugin_attr_type in required_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
exit(3)
plugin.id = plugin.name.replace(' ', '_')
self.plugins.append(plugin)
def call(self, plugin_type, request, *args, **kwargs):
ret = True
for plugin in request.user_plugins:
if hasattr(plugin, plugin_type):
ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
if not ret:
break
return ret
plugins = PluginStore()
plugins.register(self_ip)

21
searx/plugins/self_ip.py Normal file
View File

@ -0,0 +1,21 @@
from flask.ext.babel import gettext
name = "Self IP"
description = gettext('Display your source IP address if the query expression is "ip"')
default_on = True
# attach callback to the pre search hook
# request: flask request object
# ctx: the whole local context of the pre search hook
def pre_search(request, ctx):
if ctx['search'].query == 'ip':
x_forwarded_for = request.headers.getlist("X-Forwarded-For")
if x_forwarded_for:
ip = x_forwarded_for[0]
else:
ip = request.remote_addr
ctx['search'].answers.clear()
ctx['search'].answers.add(ip)
# return False prevents exeecution of the original block
return False
return True

27
searx/search.py
View File

@ -329,8 +329,8 @@ class Search(object):
self.blocked_engines = get_blocked_engines(engines, request.cookies)
self.results = []
self.suggestions = []
self.answers = []
self.suggestions = set()
self.answers = set()
self.infoboxes = []
self.request_data = {}
@ -429,9 +429,6 @@ class Search(object):
requests = []
results_queue = Queue()
results = {}
suggestions = set()
answers = set()
infoboxes = []
# increase number of searches
number_of_searches += 1
@ -511,7 +508,7 @@ class Search(object):
selected_engine['name']))
if not requests:
return results, suggestions, answers, infoboxes
return self
# send all search-request
threaded_requests(requests)
@ -519,19 +516,19 @@ class Search(object):
engine_name, engine_results = results_queue.get_nowait()
# TODO type checks
[suggestions.add(x['suggestion'])
[self.suggestions.add(x['suggestion'])
for x in list(engine_results)
if 'suggestion' in x
and engine_results.remove(x) is None]
[answers.add(x['answer'])
[self.answers.add(x['answer'])
for x in list(engine_results)
if 'answer' in x
and engine_results.remove(x) is None]
infoboxes.extend(x for x in list(engine_results)
if 'infobox' in x
and engine_results.remove(x) is None)
self.infoboxes.extend(x for x in list(engine_results)
if 'infobox' in x
and engine_results.remove(x) is None)
results[engine_name] = engine_results
@ -541,16 +538,16 @@ class Search(object):
engines[engine_name].stats['result_count'] += len(engine_results)
# score results and remove duplications
results = score_results(results)
self.results = score_results(results)
# merge infoboxes according to their ids
infoboxes = merge_infoboxes(infoboxes)
self.infoboxes = merge_infoboxes(self.infoboxes)
# update engine stats, using calculated score
for result in results:
for result in self.results:
for res_engine in result['engines']:
engines[result['engine']]\
.stats['score_count'] += result['score']
# return results, suggestions, answers and infoboxes
return results, suggestions, answers, infoboxes
return self

1
searx/settings.yml
View File

@ -106,6 +106,7 @@ engines:
- name : gigablast
engine : gigablast
shortcut : gb
disabled: True
- name : github
engine : github

8
searx/templates/oscar/macros.html
View File

@ -59,3 +59,11 @@
</div>
{% endif %}
{%- endmacro %}
{% macro checkbox_toggle(id, blocked) -%}
<div class="checkbox">
<input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />
<label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>
<label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>
</div>
{%- endmacro %}

31
searx/templates/oscar/preferences.html
View File

@ -1,4 +1,4 @@
{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %}
{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %}
{% extends "oscar/base.html" %}
{% block title %}{{ _('preferences') }} - {% endblock %}
{% block site_alert_warning_nojs %}
@ -16,6 +16,7 @@
<ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;">
<li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>
<li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>
<li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>
</ul>
<!-- Tab panes -->
@ -139,11 +140,7 @@
<div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>
{% endif %}
<div class="col-xs-6 col-sm-4 col-md-4">
<div class="checkbox">
<input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} />
<label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label>
<label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label>
</div>
{{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}
</div>
{% if rtl %}
<div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div>
@ -157,6 +154,28 @@
{% endfor %}
</div>
</div>
<div class="tab-pane active_if_nojs" id="tab_plugins">
<noscript>
<h3>{{ _('Plugins') }}</h3>
</noscript>
<fieldset>
<div class="container-fluid">
{% for plugin in plugins %}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">{{ plugin.name }}</h3>
</div>
<div class="panel-body">
<div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div>
<div class="col-xs-6 col-sm-4 col-md-6">
{{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }}
</div>
</div>
</div>
{% endfor %}
</div>
</fieldset>
</div>
</div>
<p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}
<br />

10
searx/templates/oscar/results.html
View File

@ -25,8 +25,8 @@
{% endif %}
</div>
{% endfor %}
{% if not results %}
{% if not results and not answers %}
{% include 'oscar/messages/no_results.html' %}
{% endif %}
@ -82,7 +82,7 @@
{% for infobox in infoboxes %}
{% include 'oscar/infobox.html' %}
{% endfor %}
{% endif %}
{% endif %}
{% if suggestions %}
<div class="panel panel-default">
@ -111,7 +111,7 @@
<input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}&amp;pageno={{ pageno }}{% if selected_categories %}&amp;category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly>
</div>
</form>
<label>{{ _('Download results') }}</label>
<div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
@ -122,7 +122,7 @@
<input type="hidden" name="pageno" value="{{ pageno }}">
<button type="submit" class="btn btn-default">{{ output_type }}</button>
</form>
{% endfor %}
{% endfor %}
<div class="clearfix"></div>
</div>
</div>

51
searx/tests/test_plugins.py Normal file
View File

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
from searx.testing import SearxTestCase
from searx import plugins
from mock import Mock
class PluginStoreTest(SearxTestCase):
def test_PluginStore_init(self):
store = plugins.PluginStore()
self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0)
def test_PluginStore_register(self):
store = plugins.PluginStore()
testplugin = plugins.Plugin()
store.register(testplugin)
self.assertTrue(len(store.plugins) == 1)
def test_PluginStore_call(self):
store = plugins.PluginStore()
testplugin = plugins.Plugin()
store.register(testplugin)
setattr(testplugin, 'asdf', Mock())
request = Mock(user_plugins=[])
store.call('asdf', request, Mock())
self.assertFalse(testplugin.asdf.called)
request.user_plugins.append(testplugin)
store.call('asdf', request, Mock())
self.assertTrue(testplugin.asdf.called)
class SelfIPTest(SearxTestCase):
def test_PluginStore_init(self):
store = plugins.PluginStore()
store.register(plugins.self_ip)
self.assertTrue(len(store.plugins) == 1)
request = Mock(user_plugins=store.plugins,
remote_addr='127.0.0.1')
request.headers.getlist.return_value = []
ctx = {'search': Mock(answers=set(),
query='ip')}
store.call('pre_search', request, ctx)
self.assertTrue('127.0.0.1' in ctx['search'].answers)

42
searx/tests/test_webapp.py
View File

@ -2,7 +2,6 @@
import json
from urlparse import ParseResult
from mock import patch
from searx import webapp
from searx.testing import SearxTestCase
@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase):
},
]
def search_mock(search_self, *args):
search_self.results = self.test_results
webapp.Search.search = search_mock
self.maxDiff = None # to see full diffs
def test_index_empty(self):
@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase):
self.assertEqual(result.status_code, 200)
self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)
@patch('searx.search.Search.search')
def test_index_html(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
def test_index_html(self):
result = self.app.post('/', data={'q': 'test'})
self.assertIn(
'<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa
@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase):
result.data
)
@patch('searx.search.Search.search')
def test_index_json(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
def test_index_json(self):
result = self.app.post('/', data={'q': 'test', 'format': 'json'})
result_dict = json.loads(result.data)
@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase):
self.assertEqual(
result_dict['results'][0]['url'], 'http://first.test.xyz')
@patch('searx.search.Search.search')
def test_index_csv(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
def test_index_csv(self):
result = self.app.post('/', data={'q': 'test', 'format': 'csv'})
self.assertEqual(
@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase):
result.data
)
@patch('searx.search.Search.search')
def test_index_rss(self, search):
search.return_value = (
self.test_results,
set(),
set(),
set()
)
def test_index_rss(self):
result = self.app.post('/', data={'q': 'test', 'format': 'rss'})
self.assertIn(

73
searx/webapp.py
View File

@ -27,6 +27,18 @@ import cStringIO
import os
import hashlib
from searx import logger
logger = logger.getChild('webapp')
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
from datetime import datetime, timedelta
from urllib import urlencode
from werkzeug.contrib.fixers import ProxyFix
@ -51,19 +63,9 @@ from searx.https_rewrite import https_url_rewrite
from searx.search import Search
from searx.query import Query
from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx import logger
try:
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
except:
logger.critical("cannot import dependency: pygments")
from sys import exit
exit(1)
from searx.plugins import plugins
logger = logger.getChild('webapp')
static_path, templates_path, themes =\
get_themes(settings['themes_path']
if settings.get('themes_path')
@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs):
'{}/{}'.format(kwargs['theme'], template_name), **kwargs)
@app.before_request
def pre_request():
# merge GET, POST vars
request.form = dict(request.form.items())
for k, v in request.args:
if k not in request.form:
request.form[k] = v
request.user_plugins = []
allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')
disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')
for plugin in plugins:
if ((plugin.default_on and plugin.id not in disabled_plugins)
or plugin.id in allowed_plugins):
request.user_plugins.append(plugin)
@app.route('/search', methods=['GET', 'POST'])
@app.route('/', methods=['GET', 'POST'])
def index():
@ -323,8 +342,10 @@ def index():
'index.html',
)
search.results, search.suggestions,\
search.answers, search.infoboxes = search.search(request)
if plugins.call('pre_search', request, locals()):
search.search(request)
plugins.call('post_search', request, locals())
for result in search.results:
@ -487,11 +508,11 @@ def preferences():
blocked_engines = get_blocked_engines(engines, request.cookies)
else: # on save
selected_categories = []
post_disabled_plugins = []
locale = None
autocomplete = ''
method = 'POST'
safesearch = '1'
for pd_name, pd in request.form.items():
if pd_name.startswith('category_'):
category = pd_name[9:]
@ -514,14 +535,34 @@ def preferences():
safesearch = pd
elif pd_name.startswith('engine_'):
if pd_name.find('__') > -1:
engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1)
# TODO fix underscore vs space
engine_name, category = [x.replace('_', ' ') for x in
pd_name.replace('engine_', '', 1).split('__', 1)]
if engine_name in engines and category in engines[engine_name].categories:
blocked_engines.append((engine_name, category))
elif pd_name == 'theme':
theme = pd if pd in themes else default_theme
elif pd_name.startswith('plugin_'):
plugin_id = pd_name.replace('plugin_', '', 1)
if not any(plugin.id == plugin_id for plugin in plugins):
continue
post_disabled_plugins.append(plugin_id)
else:
resp.set_cookie(pd_name, pd, max_age=cookie_max_age)
disabled_plugins = []
allowed_plugins = []
for plugin in plugins:
if plugin.default_on:
if plugin.id in post_disabled_plugins:
disabled_plugins.append(plugin.id)
elif plugin.id not in post_disabled_plugins:
allowed_plugins.append(plugin.id)
resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)
resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)
resp.set_cookie(
'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),
max_age=cookie_max_age
@ -571,6 +612,8 @@ def preferences():
autocomplete_backends=autocomplete_backends,
shortcuts={y: x for x, y in engine_shortcuts.items()},
themes=themes,
plugins=plugins,
allowed_plugins=[plugin.id for plugin in request.user_plugins],
theme=get_current_theme_name())