359 lines
11 KiB
Python
359 lines
11 KiB
Python
#!/usr/bin/env python
|
|
|
|
'''
|
|
searx is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU Affero General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
searx is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU Affero General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Affero General Public License
|
|
along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
|
|
|
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
|
'''
|
|
|
|
import json
|
|
import cStringIO
|
|
import os
|
|
|
|
from flask import Flask, request, render_template
|
|
from flask import url_for, Response, make_response, redirect
|
|
from flask import send_from_directory
|
|
|
|
from searx import settings, searx_dir
|
|
from searx.engines import search, categories, engines, get_engines_stats
|
|
from searx.utils import UnicodeWriter
|
|
from searx.utils import highlight_content, html_to_text
|
|
from searx.languages import language_codes
|
|
|
|
from flask.ext.babel import Babel
|
|
|
|
|
|
app = Flask(
|
|
__name__,
|
|
static_folder=os.path.join(searx_dir, 'static'),
|
|
template_folder=os.path.join(searx_dir, 'templates')
|
|
)
|
|
|
|
app.secret_key = settings['server']['secret_key']
|
|
|
|
babel = Babel(app)
|
|
|
|
#TODO configurable via settings.yml
|
|
favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',
|
|
'twitter', 'stackoverflow', 'github']
|
|
|
|
|
|
@babel.localeselector
|
|
def get_locale():
|
|
locale = request.accept_languages.best_match(settings['locales'].keys())
|
|
|
|
if request.cookies.get('locale', '') in settings['locales']:
|
|
locale = request.cookies.get('locale', '')
|
|
|
|
if 'locale' in request.args\
|
|
and request.args['locale'] in settings['locales']:
|
|
locale = request.args['locale']
|
|
|
|
if 'locale' in request.form\
|
|
and request.form['locale'] in settings['locales']:
|
|
locale = request.form['locale']
|
|
|
|
return locale
|
|
|
|
|
|
def get_base_url():
|
|
if settings['server']['base_url']:
|
|
hostname = settings['server']['base_url']
|
|
else:
|
|
scheme = 'http'
|
|
if request.is_secure:
|
|
scheme = 'https'
|
|
hostname = url_for('index', _external=True, _scheme=scheme)
|
|
return hostname
|
|
|
|
|
|
def render(template_name, **kwargs):
|
|
global categories
|
|
kwargs['categories'] = ['general']
|
|
kwargs['categories'].extend(x for x in
|
|
sorted(categories.keys()) if x != 'general')
|
|
if not 'selected_categories' in kwargs:
|
|
kwargs['selected_categories'] = []
|
|
cookie_categories = request.cookies.get('categories', '').split(',')
|
|
for ccateg in cookie_categories:
|
|
if ccateg in categories:
|
|
kwargs['selected_categories'].append(ccateg)
|
|
if not len(kwargs['selected_categories']):
|
|
kwargs['selected_categories'] = ['general']
|
|
return render_template(template_name, **kwargs)
|
|
|
|
|
|
def parse_query(query):
|
|
query_engines = []
|
|
query_parts = query.split()
|
|
|
|
if query_parts[0].startswith('-'):
|
|
engine_name = query_parts[0][1:].replace('_', ' ')
|
|
if engine_name in engines:
|
|
query_engines.append({'category': 'none',
|
|
'name': query_parts[0][1:]})
|
|
elif query_parts[0].startswith('!'):
|
|
category_name = query_parts[0][1:].replace('_', ' ')
|
|
if category_name in categories:
|
|
query_engines.extend({'category': category_name,
|
|
'name': engine.name}
|
|
for engine in categories[category_name])
|
|
|
|
if len(query_engines):
|
|
query = query.replace(query_parts[0], '', 1).strip()
|
|
return query, query_engines
|
|
|
|
|
|
@app.route('/', methods=['GET', 'POST'])
|
|
def index():
|
|
"""Render index page.
|
|
|
|
Supported outputs: html, json, csv, rss.
|
|
"""
|
|
paging = False
|
|
lang = 'all'
|
|
|
|
if request.cookies.get('language')\
|
|
and request.cookies['language'] in (x[0] for x in language_codes):
|
|
lang = request.cookies['language']
|
|
|
|
if request.method == 'POST':
|
|
request_data = request.form
|
|
else:
|
|
request_data = request.args
|
|
if not request_data.get('q'):
|
|
return render('index.html')
|
|
|
|
pageno_param = request_data.get('pageno', '1')
|
|
if not pageno_param.isdigit() or int(pageno_param) < 1:
|
|
return render('index.html')
|
|
|
|
pageno = int(pageno_param)
|
|
|
|
selected_categories = []
|
|
|
|
query, selected_engines = parse_query(request_data['q'].encode('utf-8'))
|
|
|
|
if len(selected_engines):
|
|
selected_categories = list(set(engine['category']
|
|
for engine in selected_engines))
|
|
else:
|
|
for pd_name, pd in request_data.items():
|
|
if pd_name.startswith('category_'):
|
|
category = pd_name[9:]
|
|
if not category in categories:
|
|
continue
|
|
selected_categories.append(category)
|
|
if not len(selected_categories):
|
|
cookie_categories = request.cookies.get('categories', '')
|
|
cookie_categories = cookie_categories.split(',')
|
|
for ccateg in cookie_categories:
|
|
if ccateg in categories:
|
|
selected_categories.append(ccateg)
|
|
if not len(selected_categories):
|
|
selected_categories = ['general']
|
|
|
|
for categ in selected_categories:
|
|
selected_engines.extend({'category': categ,
|
|
'name': x.name}
|
|
for x in categories[categ])
|
|
|
|
results, suggestions = search(query,
|
|
request,
|
|
selected_engines,
|
|
pageno,
|
|
lang)
|
|
|
|
for result in results:
|
|
if not paging and engines[result['engine']].paging:
|
|
paging = True
|
|
if request_data.get('format', 'html') == 'html':
|
|
if 'content' in result:
|
|
result['content'] = highlight_content(result['content'], query)
|
|
result['title'] = highlight_content(result['title'], query)
|
|
else:
|
|
if 'content' in result:
|
|
result['content'] = html_to_text(result['content']).strip()
|
|
result['title'] = html_to_text(result['title']).strip()
|
|
if len(result['url']) > 74:
|
|
url_parts = result['url'][:35], result['url'][-35:]
|
|
result['pretty_url'] = '{0}[...]{1}'.format(*url_parts)
|
|
else:
|
|
result['pretty_url'] = result['url']
|
|
|
|
for engine in result['engines']:
|
|
if engine in favicons:
|
|
result['favicon'] = engine
|
|
|
|
if request_data.get('format') == 'json':
|
|
return Response(json.dumps({'query': query, 'results': results}),
|
|
mimetype='application/json')
|
|
elif request_data.get('format') == 'csv':
|
|
csv = UnicodeWriter(cStringIO.StringIO())
|
|
keys = ('title', 'url', 'content', 'host', 'engine', 'score')
|
|
if len(results):
|
|
csv.writerow(keys)
|
|
for row in results:
|
|
row['host'] = row['parsed_url'].netloc
|
|
csv.writerow([row.get(key, '') for key in keys])
|
|
csv.stream.seek(0)
|
|
response = Response(csv.stream.read(), mimetype='application/csv')
|
|
content_disp = 'attachment;Filename=searx_-_{0}.csv'.format(query)
|
|
response.headers.add('Content-Disposition', content_disp)
|
|
return response
|
|
elif request_data.get('format') == 'rss':
|
|
response_rss = render(
|
|
'opensearch_response_rss.xml',
|
|
results=results,
|
|
q=request_data['q'],
|
|
number_of_results=len(results),
|
|
base_url=get_base_url()
|
|
)
|
|
return Response(response_rss, mimetype='text/xml')
|
|
|
|
return render(
|
|
'results.html',
|
|
results=results,
|
|
q=request_data['q'],
|
|
selected_categories=selected_categories,
|
|
paging=paging,
|
|
pageno=pageno,
|
|
suggestions=suggestions
|
|
)
|
|
|
|
|
|
@app.route('/about', methods=['GET'])
|
|
def about():
|
|
"""Render about page"""
|
|
return render('about.html')
|
|
|
|
|
|
@app.route('/engines', methods=['GET'])
|
|
def list_engines():
|
|
"""Render engines page.
|
|
|
|
List of all supported engines.
|
|
"""
|
|
global categories
|
|
return render('engines.html', categs=categories.items())
|
|
|
|
|
|
@app.route('/preferences', methods=['GET', 'POST'])
|
|
def preferences():
|
|
"""Render preferences page.
|
|
|
|
Settings that are going to be saved as cookies."""
|
|
lang = None
|
|
|
|
if request.cookies.get('language')\
|
|
and request.cookies['language'] in (x[0] for x in language_codes):
|
|
lang = request.cookies['language']
|
|
|
|
if request.method == 'POST':
|
|
selected_categories = []
|
|
locale = None
|
|
for pd_name, pd in request.form.items():
|
|
if pd_name.startswith('category_'):
|
|
category = pd_name[9:]
|
|
if not category in categories:
|
|
continue
|
|
selected_categories.append(category)
|
|
elif pd_name == 'locale' and pd in settings['locales']:
|
|
locale = pd
|
|
elif pd_name == 'language' and (pd == 'all' or
|
|
pd in (x[0] for
|
|
x in language_codes)):
|
|
lang = pd
|
|
|
|
resp = make_response(redirect('/'))
|
|
|
|
if locale:
|
|
# cookie max age: 4 weeks
|
|
resp.set_cookie(
|
|
'locale', locale,
|
|
max_age=60 * 60 * 24 * 7 * 4
|
|
)
|
|
|
|
if lang:
|
|
# cookie max age: 4 weeks
|
|
resp.set_cookie(
|
|
'language', lang,
|
|
max_age=60 * 60 * 24 * 7 * 4
|
|
)
|
|
|
|
if selected_categories:
|
|
# cookie max age: 4 weeks
|
|
resp.set_cookie(
|
|
'categories', ','.join(selected_categories),
|
|
max_age=60 * 60 * 24 * 7 * 4
|
|
)
|
|
return resp
|
|
return render('preferences.html',
|
|
locales=settings['locales'],
|
|
current_locale=get_locale(),
|
|
current_language=lang or 'all',
|
|
language_codes=language_codes)
|
|
|
|
|
|
@app.route('/stats', methods=['GET'])
|
|
def stats():
|
|
"""Render engine statistics page."""
|
|
global categories
|
|
stats = get_engines_stats()
|
|
return render('stats.html', stats=stats)
|
|
|
|
|
|
@app.route('/robots.txt', methods=['GET'])
|
|
def robots():
|
|
return Response("""User-agent: *
|
|
Allow: /
|
|
Allow: /about
|
|
Disallow: /stats
|
|
Disallow: /engines
|
|
""", mimetype='text/plain')
|
|
|
|
|
|
@app.route('/opensearch.xml', methods=['GET'])
|
|
def opensearch():
|
|
method = 'post'
|
|
# chrome/chromium only supports HTTP GET....
|
|
if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
|
|
method = 'get'
|
|
ret = render('opensearch.xml', method=method, host=get_base_url())
|
|
resp = Response(response=ret,
|
|
status=200,
|
|
mimetype="application/xml")
|
|
return resp
|
|
|
|
|
|
@app.route('/favicon.ico')
|
|
def favicon():
|
|
return send_from_directory(os.path.join(app.root_path, 'static/img'),
|
|
'favicon.png',
|
|
mimetype='image/vnd.microsoft.icon')
|
|
|
|
|
|
def run():
|
|
from gevent import monkey
|
|
monkey.patch_all()
|
|
|
|
app.run(
|
|
debug=settings['server']['debug'],
|
|
use_debugger=settings['server']['debug'],
|
|
port=settings['server']['port']
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|