fix yacy-engine and add comments

* add language support
* add image-support
* add basic-implementation for video, music and file handling
* refactore engine
* add comments
This commit is contained in:
Thomas Pointhuber 2014-09-03 13:47:12 +02:00
parent 63560607ad
commit 2cd980134b
2 changed files with 74 additions and 18 deletions

View File

@ -1,40 +1,89 @@
## Yacy (Web, Images, Videos, Music, Files)
#
# @website http://yacy.net
# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
#
# @using-api yes
# @results JSON
# @stable yes
# @parse (general) url, title, content, publishedDate
# @parse (images) url, title, img_src
#
# @todo parse video, audio and file results
from json import loads from json import loads
from urllib import urlencode from urllib import urlencode
from dateutil import parser
url = 'http://localhost:8090' # engine dependent config
search_url = '/yacysearch.json?{query}&maximumRecords=10' categories = ['general', 'images'] #TODO , 'music', 'videos', 'files'
paging = True
language_support = True
number_of_results = 5
# search-url
base_url = 'http://localhost:8090'
search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limit}&contentdom={search_type}&resource=global'
# yacy specific type-definitions
search_types = {'general': 'text',
'images': 'image',
'files': 'app',
'music': 'audio',
'videos': 'video'}
# do search-request
def request(query, params): def request(query, params):
params['url'] = url + search_url.format(query=urlencode({'query': query})) offset = (params['pageno'] - 1) * number_of_results
search_type = search_types.get(params['category'], '0')
params['url'] = base_url + search_url.format(query=urlencode({'query': query}),
offset=offset,
limit=number_of_results,
search_type=search_type)
# add language tag if specified
if params['language'] != 'all':
params['url'] += '&lr=lang_' + params['language'].split('_')[0]
return params return params
# get response from search-request
def response(resp): def response(resp):
results = []
raw_search_results = loads(resp.text) raw_search_results = loads(resp.text)
# return empty array if there are no results
if not raw_search_results: if not raw_search_results:
return [] return []
search_results = raw_search_results.get('channels', {})[0].get('items', []) search_results = raw_search_results.get('channels', {})[0].get('items', [])
results = [] if resp.search_params['category'] == 'general':
# parse general results
for result in search_results: for result in search_results:
tmp_result = {} publishedDate = parser.parse(result['pubDate'])
tmp_result['title'] = result['title']
tmp_result['url'] = result['link']
tmp_result['content'] = ''
if result['description']: # append result
tmp_result['content'] += result['description'] + "<br/>" results.append({'url': result['link'],
'title': result['title'],
'content': result['description'],
'publishedDate': publishedDate})
if result['pubDate']: elif resp.search_params['category'] == 'images':
tmp_result['content'] += result['pubDate'] + "<br/>" # parse image results
for result in search_results:
# append result
results.append({'url': result['url'],
'title': result['title'],
'content': '',
'img_src': result['image'],
'template': 'images.html'})
if result['size'] != '-1': #TODO parse video, audio and file results
tmp_result['content'] += result['sizename']
results.append(tmp_result)
# return results
return results return results

View File

@ -143,6 +143,13 @@ engines:
locale : en-US locale : en-US
shortcut : vm shortcut : vm
# - name : yacy
# engine : yacy
# shortcut : ya
# base_url : 'http://localhost:8090'
# number_of_results : 5
# timeout: 3.0
locales: locales:
en : English en : English
de : Deutsch de : Deutsch