# -*- coding: utf-8 -*- # This script generates languages.py from intersecting each engine's supported languages. # # Output files (engines_languages.json and languages.py) # are written in current directory to avoid overwriting in case something goes wrong. import json import io from sys import path from babel import Locale, UnknownLocaleError from babel.languages import get_global path.append('../searx') # noqa from searx import settings from searx.engines import initialize_engines, engines # Output files. engines_languages_file = 'engines_languages.json' languages_file = 'languages.py' # Fetchs supported languages for each engine and writes json file with those. def fetch_supported_languages(): engines_languages = {} names = list(engines) names.sort() for engine_name in names: print("fetching languages of engine %s" % engine_name) if hasattr(engines[engine_name], 'fetch_supported_languages'): engines_languages[engine_name] = engines[engine_name].fetch_supported_languages() if type(engines_languages[engine_name]) == list: engines_languages[engine_name] = sorted(engines_languages[engine_name]) # write json file with open(engines_languages_file, 'w', encoding='utf-8') as f: json.dump(engines_languages, f, indent=2, sort_keys=True) return engines_languages # Get babel Locale object from lang_code if possible. def get_locale(lang_code): try: locale = Locale.parse(lang_code, sep='-') return locale except (UnknownLocaleError, ValueError): return None # Append engine_name to list of engines that support locale. def add_engine_counter(lang_code, engine_name, languages): if lang_code in languages: if 'counter' not in languages[lang_code]: languages[lang_code]['counter'] = [engine_name] elif engine_name not in languages[lang_code]['counter']: languages[lang_code]['counter'].append(engine_name) # Join all language lists. # TODO: Add language names from engine's language list if name not known by babel. def join_language_lists(engines_languages): language_list = {} for engine_name in engines_languages: for lang_code in engines_languages[engine_name]: # apply custom fixes if necessary if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values(): lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items() if lang_code == alias) locale = get_locale(lang_code) # ensure that lang_code uses standard language and country codes if locale and locale.territory: lang_code = locale.language + '-' + locale.territory # add locale if it's not in list if lang_code not in language_list: if locale: language_list[lang_code] = {'name': locale.get_language_name().title(), 'english_name': locale.english_name, 'country': locale.get_territory_name() or ''} # also add language without country if locale.language not in language_list: language_list[locale.language] = {'name': locale.get_language_name().title(), 'english_name': locale.english_name} else: language_list[lang_code] = {} # count engine for both language_country combination and language alone add_engine_counter(lang_code, engine_name, language_list) add_engine_counter(lang_code.split('-')[0], engine_name, language_list) return language_list # Filter language list so it only includes the most supported languages and countries. def filter_language_list(all_languages): min_supported_engines = 10 main_engines = [engine_name for engine_name in engines.keys() if 'general' in engines[engine_name].categories and engines[engine_name].supported_languages and not engines[engine_name].disabled] # filter list to include only languages supported by most engines or all default general engines filtered_languages = {code: lang for code, lang in all_languages.items() if (len(lang.get('counter', [])) >= min_supported_engines or all(main_engine in lang.get('counter', []) for main_engine in main_engines))} return filtered_languages # Add country codes to languages without one and filter out language codes. def assign_country_codes(filtered_languages, all_languages): sorted_languages = sorted(all_languages, key=lambda lang: len(all_languages[lang].get('counter', [])), reverse=True) previous_lang = None previous_code = None countries = 0 for current_code in sorted(filtered_languages): current_lang = current_code.split('-')[0] # count country codes per language if current_lang == previous_lang: countries += 1 else: if previous_lang is not None: # if language has no single country code if countries == 0: # try to get country code with most supported engines for l in sorted_languages: l_parts = l.split('-') if len(l_parts) == 2 and l_parts[0] == previous_lang: filtered_languages[l] = all_languages[l] filtered_languages[l]['country'] = '' countries = 1 break if countries == 0: # get most likely country code from babel subtags = get_global('likely_subtags').get(previous_lang) if subtags: subtag_parts = subtags.split('_') new_code = subtag_parts[0] + '-' + subtag_parts[-1] filtered_languages[new_code] = all_languages[previous_lang] countries = 1 if countries == 1: # remove countryless version of language if there's only one country del filtered_languages[previous_lang] if previous_code in filtered_languages: filtered_languages[previous_code]['country'] = '' countries = 0 previous_lang = current_lang previous_code = current_code # Write languages.py. def write_languages_file(languages): new_file = open(languages_file, 'wb') file_content = '# -*- coding: utf-8 -*-\n'\ + '# list of language codes\n'\ + '# this file is generated automatically by utils/update_search_languages.py\n'\ + '\nlanguage_codes = (' for code in sorted(languages): file_content += '\n (u"' + code + '"'\ + ', u"' + languages[code]['name'].split(' (')[0] + '"'\ + ', u"' + languages[code].get('country', '') + '"'\ + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),' # remove last comma file_content = file_content[:-1] file_content += '\n)\n' new_file.write(file_content.encode('utf8')) new_file.close() if __name__ == "__main__": initialize_engines(settings['engines']) engines_languages = fetch_supported_languages() all_languages = join_language_lists(engines_languages) filtered_languages = filter_language_list(all_languages) assign_country_codes(filtered_languages, all_languages) write_languages_file(filtered_languages)