LibreMiami
/
libremiami-search
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1803 from return42/filtron 

Tooling Box to setup & maintain searx instances and services.
This commit is contained in:
Adam Tauber 2020-06-18 19:34:32 +02:00 committed by GitHub
parent 0dc5bdb6ef 0616684baa
commit 4a8a16559a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
65 changed files with 7275 additions and 520 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.config.sh Normal file
View File

@ -0,0 +1,55 @@
# -*- coding: utf-8; mode: sh -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck shell=bash disable=SC2034
#
# This environment is used by ./utils scripts like filtron.sh or searx.sh. The
# default values are *most flexible* and *best maintained*, you normally not
# need to change the defaults (except PUBLIC_URL).
#
# Before you change any value here you have to uninstall any previous
# installation. Further is it recommended to backup your changes simply by
# adding them to you local brand (git branch)::
#
# git add .config
# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx"
# The default is taken from ./utils/brand.env.
PUBLIC_URL="${SEARX_URL}"
if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then
# hint: Linux containers do not have DNS entries, lets use IPs
PUBLIC_URL="http://$(primary_ip)/searx"
fi
# searx.sh
# ---------
# SEARX_INTERNAL_URL="127.0.0.1:8888"
# Only change, if you maintain a searx brand in your searx fork.
# GIT_BRANCH="${GIT_BRANCH:-master}"
# filtron.sh
# ----------
# FILTRON_API="127.0.0.1:4005"
# FILTRON_LISTEN="127.0.0.1:4004"
# FILTRON_TARGET="127.0.0.1:8888"
# morty.sh
# --------
# morty listen address
# MORTY_LISTEN="127.0.0.1:3000"
# PUBLIC_URL_PATH_MORTY="/morty/"
# system services
# ---------------
# Common $HOME folder of the service accounts
# SERVICE_HOME_BASE="/usr/local"
# **experimental**: Set SERVICE_USER to run all services by one account, but be
# aware that removing discrete components might conflict!
# SERVICE_USER=searx

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
# to sync with .dockerignore
.coverage
coverage/
cache/
.installed.cfg
engines.cfg
env

65
Makefile
View File

@ -1,24 +1,31 @@
# -*- coding: utf-8; mode: makefile-gmake -*-
.DEFAULT_GOAL=help
# START Makefile setup
export GIT_URL=https://github.com/asciimoo/searx
export GIT_BRANCH=master
export SEARX_URL=https://searx.me
export DOCS_URL=https://asciimoo.github.io/searx
# END Makefile setup
include utils/makefile.include
PYOBJECTS = searx
DOC = docs
PY_SETUP_EXTRAS ?= \[test\]
PYDIST=./dist/py
PYBUILD=./build/py
include utils/makefile.include
include utils/makefile.python
include utils/makefile.sphinx
all: clean install
PHONY += help
help:
PHONY += help-min help-all help
help: help-min
@echo ''
@echo 'to get more help: make help-all'
help-min:
@echo ' test - run developer tests'
@echo ' docs - build documentation'
@echo ' docs-live - autobuild HTML documentation while editing'
@ -33,9 +40,18 @@ help:
@echo ' docker - build Docker image'
@echo ' node.env - download & install npm dependencies locally'
@echo ''
@$(MAKE) -s -f utils/makefile.include make-help
@echo 'environment'
@echo ' SEARX_URL = $(SEARX_URL)'
@echo ' GIT_URL = $(GIT_URL)'
@echo ' DOCS_URL = $(DOCS_URL)'
@echo ''
@$(MAKE) -s -f utils/makefile.python python-help
@$(MAKE) -e -s make-help
help-all: help-min
@echo ''
@$(MAKE) -e -s python-help
@echo ''
@$(MAKE) -e -s docs-help
PHONY += install
install: buildenv pyenvinstall
@ -44,7 +60,7 @@ PHONY += uninstall
uninstall: pyenvuninstall
PHONY += clean
clean: pyclean node.clean test.clean
clean: pyclean docs-clean node.clean test.clean
$(call cmd,common_clean)
PHONY += run
@ -61,14 +77,24 @@ run: buildenv pyenvinstall
# docs
# ----
sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
PHONY += docs
docs: buildenv pyenvinstall sphinx-doc
docs: sphinx-doc-prebuilds sphinx-doc
$(call cmd,sphinx,html,docs,docs)
PHONY += docs-live
docs-live: buildenv pyenvinstall sphinx-live
docs-live: sphinx-doc-prebuilds sphinx-live
$(call cmd,sphinx_autobuild,html,docs,docs)
PHONY += prebuild-includes
prebuild-includes:
$(Q)mkdir -p $(DOCS_BUILD)/includes
$(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
$(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
$(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
$(GH_PAGES)::
@echo "doc available at --> $(DOCS_URL)"
@ -94,12 +120,14 @@ useragents.update: pyenvinstall
buildenv:
$(Q)echo "build searx/brand.py"
$(Q)echo "GIT_URL = '$(GIT_URL)'" > searx/brand.py
$(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'" >> searx/brand.py
$(Q)echo "ISSUE_URL = 'https://github.com/asciimoo/searx/issues'" >> searx/brand.py
$(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py
$(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py
$(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py
$(Q)echo "build utils/brand.env"
$(Q)echo "export GIT_URL='$(GIT_URL)'" > utils/brand.env
$(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'" >> utils/brand.env
$(Q)echo "export ISSUE_URL='https://github.com/asciimoo/searx/issues'" >> utils/brand.env
$(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env
$(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env
@ -182,8 +210,7 @@ gecko.driver:
# test
# ----
PHONY += test test.pylint test.pep8 test.unit test.coverage test.robot
PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
ifeq ($(PY),2)
@ -191,6 +218,7 @@ test.pylint:
@echo "LINT skip liniting py2"
else
# TODO: balance linting with pylint
test.pylint: pyenvinstall
$(call cmd,pylint,\
searx/preferences.py \
@ -202,6 +230,17 @@ endif
# E402 module level import not at top of file
# W503 line break before binary operator
# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
test.sh:
shellcheck -x -s bash utils/brand.env
shellcheck -x utils/lib.sh
shellcheck -x utils/filtron.sh
shellcheck -x utils/searx.sh
shellcheck -x utils/morty.sh
shellcheck -x utils/lxc.sh
shellcheck -x utils/lxc-searx.env
shellcheck -x .config.sh
test.pep8: pyenvinstall
@echo "TEST pep8"
$(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests

41
docs/_themes/searx/static/searx.css vendored
View File

@ -33,7 +33,7 @@ p.sidebar-title, .sidebar p {
/* admonitions
*/
div.admonition, div.topic {
div.admonition, div.topic, div.toctree-wrapper {
background-color: #fafafa;
margin: 8px 0px;
padding: 1em;
@ -42,6 +42,16 @@ div.admonition, div.topic {
border-right: none;
border-bottom: none;
border-left: 5pt solid #ccc;
list-style-type: disclosure-closed;
}
div.toctree-wrapper p.caption {
font-weight: normal;
font-size: 24px;
margin: 0 0 10px 0;
padding: 0;
line-height: 1;
display: inline;
}
p.admonition-title:after {
@ -128,3 +138,32 @@ caption {
caption-side: top;
text-align: left;
}
/* bugs since sphinx 3.1
See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS:
- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605
- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972
*/
li > p:first-child {
margin-top: 0;
}
li > p:last-child {
margin-bottom: 0;
}
div.admonition dl {
margin-bottom: 0;
}
div.sidebar {
clear: none;
}
div.admonition, div.topic, pre {
clear: none;
}

8
docs/admin/arch_public.dot
View File

@ -4,11 +4,11 @@ digraph G {
edge [fontname="Sans"];
browser [label="Browser", shape=Mdiamond];
rp [label="Reverse Proxy", href="url to configure reverse proxy"];
filtron [label="Filtron", href="https://github.com/asciimoo/filtron"];
morty [label="Morty", href="https://github.com/asciimoo/morty"];
rp [label="Reverse Proxy", href="https://asciimoo.github.io/searx/utils/filtron.sh.html#public-reverse-proxy"];
filtron [label="Filtron", href="https://asciimoo.github.io/searx/utils/filtron.sh.html"];
morty [label="Morty", href="https://asciimoo.github.io/searx/utils/morty.sh.html"];
static [label="Static files", href="url to configure static files"];
uwsgi [label="uwsgi", href="url to configure uwsgi"]
uwsgi [label="uwsgi", href="https://asciimoo.github.io/searx/utils/searx.sh.html"]
searx1 [label="Searx #1"];
searx2 [label="Searx #2"];
searx3 [label="Searx #3"];

14
docs/admin/architecture.rst
View File

@ -4,17 +4,21 @@
Architecture
============
.. sidebar:: Needs work!
.. sidebar:: Further reading
This article needs some work / Searx is a collaborative effort. If you have
any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
:ref:`how to contribute`.
- Reverse Proxy: :ref:`Apache <apache searx site>` & :ref:`nginx <nginx searx
site>`
- Filtron: :ref:`searx filtron`
- Morty: :ref:`searx morty`
- uWSGI: :ref:`searx uwsgi`
- Searx: :ref:`installation basic`
Herein you will find some hints and suggestions about typical architectures of
searx infrastructures.
We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
It shows a *reference* setup for public searx instances.
It shows a *reference* setup for public searx instances which can build up and
maintained by the scripts from our :ref:`toolboxing`.
.. _arch public:

74
docs/admin/buildhosts.rst
View File

@ -9,8 +9,27 @@ Buildhosts
If you have any contribution send us your :pull:`PR <../pulls>`, see
:ref:`how to contribute`.
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
To get best results from build, its recommend to install additional packages
on build hosts.
on build hosts (see :ref:`searx.sh`).::
sudo -H ./utils/searx.sh install buildhost
This will install packages needed by searx:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START distro-packages
:end-before: END distro-packages
and packages needed to build docuemtation and run tests:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START build-packages
:end-before: END build-packages
.. _docs build:
@ -35,8 +54,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF
creation, its also needed for :ref:`math` when HTML output is build.
To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
as images (``sphinx.ext.imgmath`` extension). If your docs build (``make
docs``) shows warnings like this::
as images (``sphinx.ext.imgmath`` extension).
Here is the extract from the :origin:`docs/conf.py` file, setting math renderer
to ``imgmath``:
.. literalinclude:: ../conf.py
:language: python
:start-after: # sphinx.ext.imgmath setup
:end-before: # sphinx.ext.imgmath setup END
If your docs build (``make docs``) shows warnings like this::
WARNING: dot(1) not found, for better output quality install \
graphviz from http://www.graphviz.org
@ -47,8 +75,6 @@ docs``) shows warnings like this::
you need to install additional packages on your build host, to get better HTML
output.
.. _system requirements:
.. tabs::
.. group-tab:: Ubuntu / debian
@ -92,12 +118,38 @@ For PDF output you also need:
$ sudo dnf install \
texlive-collection-fontsrecommended texlive-collection-latex \
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \
ImageMagick
.. _system requirements END:
.. _sh lint:
.. literalinclude:: ../conf.py
:language: python
:start-after: # sphinx.ext.imgmath setup
:end-before: # sphinx.ext.imgmath setup END
Lint shell scripts
==================
.. _ShellCheck: https://github.com/koalaman/shellcheck
To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool.
.. SNIP sh lint requirements
.. tabs::
.. group-tab:: Ubuntu / debian
.. code-block:: sh
$ sudo apt install shellcheck
.. group-tab:: Arch Linux
.. code-block:: sh
$ sudo pacman -S shellcheck
.. group-tab:: Fedora / RHEL
.. code-block:: sh
$ sudo dnf install ShellCheck
.. SNAP sh lint requirements

247
docs/admin/filtron.rst
View File

@ -1,18 +1,51 @@
.. _searx filtron:
==========================
How to protect an instance
==========================
Searx depens on external search services. To avoid the abuse of these services
.. sidebar:: further reading
- :ref:`filtron.sh`
- :ref:`nginx searx site`
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
.. _filtron: https://github.com/asciimoo/filtron
Searx depends on external search services. To avoid the abuse of these services
it is advised to limit the number of requests processed by searx.
An application firewall, ``filtron`` solves exactly this problem. Information
on how to install it can be found at the `project page of filtron
<https://github.com/asciimoo/filtron>`__.
An application firewall, filtron_ solves exactly this problem. Filtron is just
a middleware between your web server (nginx, apache, ...) and searx, we describe
such infratructures in chapter: :ref:`architecture`.
filtron & go
============
.. _Go: https://golang.org/
.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md
Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply
installed by ``go get`` package management (see `filtron README`_). If you use
filtron as middleware, a more isolated setup is recommended. To simplify such
an installation and the maintenance of, use our script :ref:`filtron.sh`.
.. _Sample configuration of filtron:
Sample configuration of filtron
===============================
.. sidebar:: Tooling box
- :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
An example configuration can be find below. This configuration limits the access
of:
@ -24,105 +57,104 @@ of:
.. code:: json
[{
"name":"search request",
"filters":[
"Param:q",
"Path=^(/|/search)$"
],
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"subrules":[
{
"name":"roboagent limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"filters":[
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
[
{
"name": "search request",
"filters": [
"Param:q",
"Path=^(/|/search)$"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
"interval": "<time-interval-in-sec (int)>"
"limit": "<max-request-number-in-interval (int)>",
"subrules": [
{
"name": "missing Accept-Language",
"filters": ["!Header:Accept-Language"],
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "suspiciously Connection=close header",
"filters": ["Header:Connection=close"],
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "IP limit",
"interval": "<time-interval-in-sec (int)>"
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "rss/json limit",
"filters": [
"Param:format=(csv|json|rss)"
],
"interval": "<time-interval-in-sec (int)>"
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "useragent limit",
"interval": "<time-interval-in-sec (int)>"
"limit": "<max-request-number-in-interval (int)>",
"aggregations": [
"Header:User-Agent"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
}
]
},
{
"name":"botlimit",
"limit":0,
"stop":true,
"filters":[
"Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"IP limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"stop":true,
"aggregations":[
"Header:X-Forwarded-For"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"rss/json limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"stop":true,
"filters":[
"Param:format=(csv|json|rss)"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
},
{
"name":"useragent limit",
"interval":"<time-interval-in-sec (int)>",
"limit":"<max-request-number-in-interval (int)>",
"aggregations":[
"Header:User-Agent"
],
"actions":[
{
"name":"block",
"params":{
"message":"Rate limit exceeded"
}
}
]
}
]
}]
}
]
.. _filtron route request:
Route request through filtron
=============================
.. sidebar:: further reading
- :ref:`filtron.sh overview`
- :ref:`installation nginx`
- :ref:`installation apache`
Filtron can be started using the following command:
.. code:: sh
@ -136,13 +168,24 @@ Use it along with ``nginx`` with the following example configuration.
.. code:: nginx
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_pass http://127.0.0.1:4004/;
# https://example.org/searx
location /searx {
proxy_pass http://127.0.0.1:4004/;
proxy_set_header Host $http_host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
}
location /searx/static {
/usr/local/searx/searx-src/searx/static;
}
Requests are coming from port 4004 going through filtron and then forwarded to
port 8888 where a searx is being run.
port 8888 where a searx is being run. For a complete setup see: :ref:`nginx
searx site`.

9
docs/admin/index.rst
View File

@ -3,9 +3,16 @@ Administrator documentation
===========================
.. toctree::
:maxdepth: 1
:maxdepth: 2
:caption: Contents
installation
installation-searx
installation-uwsgi
installation-nginx
installation-apache
installation-docker
update-searx
settings
api
architecture

514
docs/admin/installation-apache.rst Normal file
View File

@ -0,0 +1,514 @@
.. _installation apache:
===================
Install with apache
===================
.. _Apache: https://httpd.apache.org/
.. _Apache Debian:
https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x):
.. _README.Debian:
https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian
.. _Apache Arch Linux:
https://wiki.archlinux.org/index.php/Apache_HTTP_Server
.. _Apache Fedora:
https://docs.fedoraproject.org/en-US/quick-docs/getting-started-with-apache-http-server/index.html
.. _Apache directives:
https://httpd.apache.org/docs/trunk/mod/directives.html
.. _Getting Started:
https://httpd.apache.org/docs/current/en/getting-started.html
.. _Terms Used to Describe Directives:
https://httpd.apache.org/docs/current/en/mod/directive-dict.html
.. _Configuration Files:
https://httpd.apache.org/docs/current/en/configuring.html
.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost
.. _LoadModule:
https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule
.. _DocumentRoot:
https://httpd.apache.org/docs/trunk/mod/core.html#documentroot
.. _Location:
https://httpd.apache.org/docs/trunk/mod/core.html#location
.. _uWSGI Apache support:
https://uwsgi-docs.readthedocs.io/en/latest/Apache.html
.. _mod_proxy_uwsgi:
https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi
.. sidebar:: further read
- `Apache Arch Linux`_
- `Apache Debian`_ and `README.Debian`_
- `Apache Fedora`_
- `Apache directives`_
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
----
**Install** :ref:`apache searx site` using :ref:`filtron.sh <filtron.sh overview>`
.. code:: bash
$ sudo -H ./utils/filtron.sh apache install
**Install** :ref:`apache searx site` using :ref:`morty.sh <morty.sh overview>`
.. code:: bash
$ sudo -H ./utils/morty.sh apache install
----
The apache HTTP server
======================
If Apache_ is not installed, install it now. If apache_ is new to you, the
`Getting Started`_, `Configuration Files`_ and `Terms Used to Describe
Directives`_ documentation gives first orientation. There is also a list of
`Apache directives`_ *to keep in the pocket*.
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H apt-get install apache2
.. group-tab:: Arch Linux
.. code:: sh
sudo -H pacman -S apache
sudo -H systemctl enable httpd
sudo -H systemctl start http
.. group-tab:: Fedora / RHEL
.. code:: sh
sudo -H dnf install httpd
sudo -H systemctl enable httpd
sudo -H systemctl start httpd
Now at http://localhost you should see any kind of *Welcome* or *Test* page.
How this default intro site is configured, depends on the linux distribution
(compare `Apache directives`_).
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
less /etc/apache2/sites-enabled/000-default.conf
In this file, there is a line setting the `DocumentRoot`_ directive:
.. code:: apache
DocumentRoot /var/www/html
And the *welcome* page is the HTML file at ``/var/www/html/index.html``.
.. group-tab:: Arch Linux
.. code:: sh
less /etc/httpd/conf/httpd.conf
In this file, there is a line setting the `DocumentRoot`_ directive:
.. code:: apache
DocumentRoot "/srv/http"
<Directory "/srv/http">
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
The *welcome* page of Arch Linux is a page showing directory located at
``DocumentRoot``. This is *directory* page is generated by the Module
`mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_:
.. code:: apache
LoadModule autoindex_module modules/mod_autoindex.so
...
Include conf/extra/httpd-autoindex.conf
.. group-tab:: Fedora / RHEL
.. code:: sh
less /etc/httpd/conf/httpd.conf
In this file, there is a line setting the ``DocumentRoot`` directive:
.. code:: apache
DocumentRoot "/var/www/html"
...
<Directory "/var/www">
AllowOverride None
# Allow open access:
Require all granted
</Directory>
On fresh installations, the ``/var/www`` is empty and the *default
welcome page* is shown, the configuration is located at::
less /etc/httpd/conf.d/welcome.conf
.. _apache searx site:
Apache Reverse Proxy
====================
.. sidebar:: public to the internet?
If your searx instance is public, stop here and first install :ref:`filtron
reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
:ref:`installation scripts`. If already done, follow setup: *searx via
filtron plus morty*.
To setup a Apache revers proxy you have to enable the *headers* and *proxy*
modules and create a `Location`_ configuration for the searx site. In most
distributions you have to un-comment the lines in the main configuration file,
except in :ref:`The Debian Layout`.
To pass the HTTP HOST header
With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the
proxied host.
.. tabs::
.. group-tab:: Ubuntu / debian
In the Apache setup, enable headers and proxy modules:
.. code:: sh
sudo -H a2enmod headers
sudo -H a2enmod proxy
sudo -H a2enmod proxy_http
In :ref:`The Debian Layout` you create a ``searx.conf`` with the
``<Location /searx >`` directive and save this file in the *sites
available* folder at ``/etc/apache2/sites-available``. To enable the
``searx.conf`` use :man:`a2ensite`:
.. code:: sh
sudo -H a2ensite searx.conf
.. group-tab:: Arch Linux
In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
modules (LoadModule_):
.. code:: apache
FIXME needs test
LoadModule headers_module modules/mod_headers.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
.. group-tab:: Fedora / RHEL
In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
modules (LoadModule_):
.. code:: apache
FIXME needs test
LoadModule headers_module modules/mod_headers.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
.. tabs::
.. group-tab:: searx via filtron plus morty
Use this setup, if your instance is public to the internet, compare
figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
*localhost 4004* (:ref:`filtron route request`):
.. code:: apache
<Location /searx >
# SetEnvIf Request_URI "/searx" dontlog
# CustomLog /dev/null combined env=dontlog
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass http://127.0.0.1:4004
RequestHeader set X-Script-Name /searx
</Location>
2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on
*localhost 3000*
.. code:: apache
ProxyPreserveHost On
<Location /morty >
# SetEnvIf Request_URI "/morty" dontlog
# CustomLog /dev/null combined env=dontlog
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPass http://127.0.0.1:3000
RequestHeader set X-Script-Name /morty
</Location>
Note that reverse proxy advised to be used in case of single-user or
low-traffic instances. For a fully result proxification add :ref:`morty's
<searx morty>` **public URL** to your :origin:`searx/settings.yml`:
.. code:: yaml
result_proxy:
# replace example.org with your server's public name
url : https://example.org/morty
server:
image_proxy : True
uWSGI support
=============
Be warned, with this setup, your instance isn't :ref:`protected <searx
filtron>`, nevertheless it is good enough for intranet usage. In modern Linux
distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache
package and you need to install only the :ref:`uWSGI <searx uwsgi>` package:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H apt-get install uwsgi
# Ubuntu =< 18.04
sudo -H apt-get install libapache2-mod-proxy-uwsgi
.. group-tab:: Arch Linux
.. code:: sh
sudo -H pacman -S uwsgi
.. group-tab:: Fedora / RHEL
.. code:: sh
sudo -H dnf install uwsgi
The next example shows a configuration using the `uWSGI Apache support`_ via
unix sockets and `mod_proxy_uwsgi`_.
For socket communication, you have to activate ``socket =
/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888``
configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not
already exists, create a folder for the unix sockets, which can be used by the
searx account (see :ref:`create searx user`):
.. code:: bash
sudo -H mkdir -p /run/uwsgi/app/searx/
sudo -H chown -R searx:searx /run/uwsgi/app/searx/
If the server is public; to limit access to your intranet replace ``Allow from
all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class.
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: apache
LoadModule headers_module /usr/lib/apache2/mod_headers.so
LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so
# SetEnvIf Request_URI /searx dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
</Location>
.. group-tab:: Arch Linux
.. code:: apache
FIXME needs test
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
# SetEnvIf Request_URI /searx dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
</Location>
.. group-tab:: Fedora / RHEL
.. code:: apache
FIXME needs test
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
<IfModule proxy_uwsgi_module>
# SetEnvIf Request_URI /searx dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
</Location>
</IfModule>
.. group-tab:: old mod_wsgi
We show this only for historical reasons, DON'T USE `mod_uwsgi
<https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_.
ANYMORE!
.. code:: apache
<IfModule mod_uwsgi.c>
# SetEnvIf Request_URI "/searx" dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx >
Require all granted
Options FollowSymLinks Indexes
SetHandler uwsgi-handler
uWSGISocket /run/uwsgi/app/searx/socket
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
</Location>
</IfModule>
.. _restart apache:
Restart service
===============
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H systemctl restart apache2
sudo -H service uwsgi restart searx
.. group-tab:: Arch Linux
.. code:: sh
sudo -H systemctl restart httpd
sudo -H systemctl restart uwsgi@searx
.. group-tab:: Fedora / RHEL
.. code:: sh
sudo -H systemctl restart httpd
sudo -H touch /etc/uwsgi.d/searx.ini
disable logs
============
For better privacy you can disable Apache logs. In the examples above activate
one of the lines and `restart apache`_::
# SetEnvIf Request_URI "/searx" dontlog
# CustomLog /dev/null combined env=dontlog
The ``CustomLog`` directive disable logs for the whole (virtual) server, use it
when the URL of the service does not have a path component (``/searx``) / is
located at root (``/``).
.. _The Debian Layout:
The Debian Layout
=================
Be aware that the Debian layout is quite different from the standard Apache
configuration. For details look at the README.Debian_
(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on
Debian:
* :man:`apache2ctl`: Apache HTTP server control interface
* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules
* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations
* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites

28
docs/admin/installation-docker.rst Normal file
View File

@ -0,0 +1,28 @@
.. _installation docker:
===================
Docker installation
===================
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
Make sure you have installed Docker. For instance, you can deploy searx like this:
.. code:: sh
docker pull wonderfall/searx
docker run -d --name searx -p $PORT:8888 wonderfall/searx
Go to ``http://localhost:$PORT``.
See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
possible to build searx from the embedded Dockerfile.
.. code:: sh
git clone https://github.com/asciimoo/searx.git
cd searx
docker build -t whatever/searx .

381
docs/admin/installation-nginx.rst Normal file
View File

@ -0,0 +1,381 @@
.. _installation nginx:
==================
Install with nginx
==================
.. _nginx:
https://docs.nginx.com/nginx/admin-guide/
.. _nginx server configuration:
https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers
.. _nginx beginners guide:
http://nginx.org/en/docs/beginners_guide.html
.. _Getting Started wiki:
https://www.nginx.com/resources/wiki/start/
.. _uWSGI support from nginx:
https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html
.. _uwsgi_params:
https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#configuring-nginx
.. _SCRIPT_NAME:
https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name
.. sidebar:: further reading
- nginx_
- `nginx beginners guide`_
- `nginx server configuration`_
- `Getting Started wiki`_
- `uWSGI support from nginx`_
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
----
**Install** :ref:`nginx searx site` using :ref:`filtron.sh <filtron.sh overview>`
.. code:: bash
$ sudo -H ./utils/filtron.sh nginx install
**Install** :ref:`nginx searx site` using :ref:`morty.sh <morty.sh overview>`
.. code:: bash
$ sudo -H ./utils/morty.sh nginx install
----
The nginx HTTP server
=====================
If nginx_ is not installed (uwsgi will not work with the package nginx-light),
install it now.
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H apt-get install nginx
.. group-tab:: Arch Linux
.. code-block:: sh
sudo -H pacman -S nginx-mainline
sudo -H systemctl enable nginx
sudo -H systemctl start nginx
.. group-tab:: Fedora / RHEL
.. code-block:: sh
sudo -H dnf install nginx
sudo -H systemctl enable nginx
sudo -H systemctl start nginx
Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you
see a *Fedora Webserver - Test Page*. The test page comes from the default
`nginx server configuration`_. How this default intro site is configured,
depends on the linux distribution:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
less /etc/nginx/nginx.conf
there is a line including site configurations from:
.. code:: nginx
include /etc/nginx/sites-enabled/*;
.. group-tab:: Arch Linux
.. code-block:: sh
less /etc/nginx/nginx.conf
in there is a configuration section named ``server``:
.. code-block:: nginx
server {
listen 80;
server_name localhost;
# ...
}
.. group-tab:: Fedora / RHEL
.. code-block:: sh
less /etc/nginx/nginx.conf
there is a line including site configurations from:
.. code:: nginx
include /etc/nginx/conf.d/*.conf;
.. _nginx searx site:
A nginx searx site
==================
.. sidebar:: public to the internet?
If your searx instance is public, stop here and first install :ref:`filtron
reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
:ref:`installation scripts`. If already done, follow setup: *searx via
filtron plus morty*.
Now you have to create a configuration for the searx site. If nginx_ is new to
you, the `nginx beginners guide`_ is a good starting point and the `Getting
Started wiki`_ is always a good resource *to keep in the pocket*.
.. tabs::
.. group-tab:: Ubuntu / debian
Create configuration at ``/etc/nginx/sites-available/searx`` and place a
symlink to sites-enabled:
.. code:: sh
sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
.. group-tab:: Arch Linux
In the ``/etc/nginx/nginx.conf`` file, replace the configuration section
named ``server``.
.. group-tab:: Fedora / RHEL
Create configuration at ``/etc/nginx/conf.d/searx`` and place a
symlink to sites-enabled:
.. tabs::
.. group-tab:: searx via filtron plus morty
Use this setup, if your instance is public to the internet, compare
figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
*localhost 4004* (:ref:`filtron route request`):
.. code:: nginx
# https://example.org/searx
location /searx {
proxy_pass http://127.0.0.1:4004/;
proxy_set_header Host $http_host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
}
location /searx/static {
/usr/local/searx/searx-src/searx/static;
}
2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on
*localhost 3000*:
.. code:: nginx
# https://example.org/morty
location /morty {
proxy_pass http://127.0.0.1:3000/;
proxy_set_header Host $http_host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
}
Note that reverse proxy advised to be used in case of single-user or
low-traffic instances. For a fully result proxification add :ref:`morty's
<searx morty>` **public URL** to your :origin:`searx/settings.yml`:
.. code:: yaml
result_proxy:
# replace example.org with your server's public name
url : https://example.org/morty
server:
image_proxy : True
.. group-tab:: proxy or uWSGI
Be warned, with this setup, your instance isn't :ref:`protected <searx
filtron>`. Nevertheless it is good enough for intranet usage and it is a
excellent example of; *how different services can be set up*. The next
example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI
application <uwsgi configuration>`, listening on ``http =
127.0.0.1:8888``.
.. code:: nginx
# https://hostname.local/
location / {
proxy_pass http://127.0.0.1:8888;
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_buffering off;
}
Alternatively you can use the `uWSGI support from nginx`_ via unix
sockets. For socket communication, you have to activate ``socket =
/run/uwsgi/app/searx/socket`` and comment out the ``http =
127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi
configuration>`.
The example shows a nginx virtual ``server`` configuration, listening on
port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at
location ``/`` by importing the `uwsgi_params`_ and passing requests to
the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the
:ref:`searx-src clone <searx-src>` and wraps directly the
:origin:`searx/static/` content at ``location /static``.
.. code:: nginx
server {
# replace hostname.local with your server's name
server_name hostname.local;
listen 80;
listen [::]:80;
location / {
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
root /usr/local/searx/searx-src/searx;
location /static { }
}
If not already exists, create a folder for the unix sockets, which can be
used by the searx account:
.. code:: bash
mkdir -p /run/uwsgi/app/searx/
sudo -H chown -R searx:searx /run/uwsgi/app/searx/
.. group-tab:: \.\. at subdir URL
Be warned, with these setups, your instance isn't :ref:`protected <searx
filtron>`. The examples are just here to demonstrate how to export the
searx application from a subdirectory URL ``https://example.org/searx/``.
.. code:: nginx
# https://hostname.local/searx
location /searx {
proxy_pass http://127.0.0.1:8888;
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
proxy_buffering off;
}
location /searx/static {
alias /usr/local/searx/searx-src/searx/static;
}
The ``X-Script-Name /searx`` is needed by the searx implementation to
calculate relative URLs correct. The next example shows a uWSGI
configuration. Since there are no HTTP headers in a (u)WSGI protocol, the
value is shipped via the SCRIPT_NAME_ in the WSGI environment.
.. code:: nginx
# https://hostname.local/searx
location /searx {
uwsgi_param SCRIPT_NAME /searx;
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
location /searx/static {
alias /usr/local/searx/searx-src/searx;
}
For searx to work correctly the ``base_url`` must be set in the
:origin:`searx/settings.yml`.
.. code:: yaml
server:
# replace example.org with your server's public name
base_url : https://example.org/searx/
Restart service:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H systemctl restart nginx
sudo -H service uwsgi restart searx
.. group-tab:: Arch Linux
.. code:: sh
sudo -H systemctl restart nginx
sudo -H systemctl restart uwsgi@searx
.. group-tab:: Fedora
.. code:: sh
sudo -H systemctl restart nginx
sudo -H touch /etc/uwsgi.d/searx.ini
Disable logs
============
For better privacy you can disable nginx logs in ``/etc/nginx/nginx.conf``.
.. code:: nginx
http {
# ...
access_log /dev/null;
error_log /dev/null;
# ...
}

92
docs/admin/installation-searx.rst Normal file
View File

@ -0,0 +1,92 @@
.. _installation basic:
=========================
Step by step installation
=========================
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
Step by step installation with virtualenv. For Ubuntu, be sure to have enable
universe repository.
.. _install packages:
Install packages
================
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START distro-packages
:end-before: END distro-packages
.. hint::
This installs also the packages needed by :ref:`searx uwsgi`
.. _create searx user:
Create user
===========
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START create user
:end-before: END create user
.. _searx-src:
install searx & dependencies
============================
Start a interactive shell from new created user and clone searx:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START clone searx
:end-before: END clone searx
In the same shell create *virtualenv*:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START create virtualenv
:end-before: END create virtualenv
To install searx's dependencies, exit the searx *bash* session you opened above
and restart a new. Before install, first check if your *virualenv* was sourced
from the login (*~/.profile*):
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START manage.sh update_packages
:end-before: END manage.sh update_packages
.. tip::
Open a second terminal for the configuration tasks and left the ``(searx)$``
terminal open for the tasks below.
Configuration
==============
Create a copy of the :origin:`searx/settings.yml` configuration file in system's
*/etc* folder. Configure like shown below -- replace ``searx@\$(uname -n)`` with
a name of your choice -- *and/or* edit ``/etc/searx/settings.yml`` if necessary.
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START searx config
:end-before: END searx config
Check
=====
To check your searx setup, optional enable debugging and start the *webapp*.
Searx looks at the exported environment ``$SEARX_SETTINGS_PATH`` for a
configuration file.
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START check searx installation
:end-before: END check searx installation
If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the
debug option in ``settings.yml``. You can now exit searx user bash (enter exit
command twice). At this point searx is not demonized; uwsgi allows this.

149
docs/admin/installation-uwsgi.rst Normal file
View File

@ -0,0 +1,149 @@
.. _searx uwsgi:
=====
uwsgi
=====
.. sidebar:: further reading
- `systemd.unit`_
- `uWSGI Emperor`_
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
.. _One service per app in systemd:
https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
.. _uWSGI Emperor:
https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
.. _uwsgi ini file:
https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files
.. _systemd unit template:
http://0pointer.de/blog/projects/instances.html
Origin uWSGI
============
How uWSGI is implemented by distributors is different. uWSGI itself
recommend two methods
`systemd.unit`_ template files as described here `One service per app in systemd`_.
There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app
placed at dedicated locations. Take archlinux and a searx.ini as example::
unit template --> /usr/lib/systemd/system/uwsgi@.service
uwsgi ini files --> /etc/uwsgi/searx.ini
The searx app can be maintained as know from common systemd units::
systemctl enable uwsgi@searx
systemctl start uwsgi@searx
systemctl restart uwsgi@searx
systemctl stop uwsgi@searx
The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps.
The Emperor mode is a special uWSGI instance that will monitor specific
events. The Emperor mode (service) is started by a (common, not template)
systemd unit. The Emperor service will scan specific directories for `uwsgi
ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the
timestamp is modified, a corresponding action takes place: a new uWSGI
instance is started, reload or stopped. Take Fedora and a searx.ini as
example::
to start a new searx instance create --> /etc/uwsgi.d/searx.ini
to reload the instance edit timestamp --> touch /etc/uwsgi.d/searx.ini
to stop instance remove ini --> rm /etc/uwsgi.d/searx.ini
Distributors
============
The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors
mostly offer their users, even if they differ in the way they implement both
modes and their defaults. Another point they might differ is the packaging of
plugins (if so, compare :ref:`install packages`) and what the default python
interpreter is (python2 vs. python3).
Fedora starts a Emperor by default, while archlinux does not start any uwsgi
service by default. Worth to know; debian (ubuntu) follow a complete different
approach. *debian*: your are familiar with the apache infrastructure? .. they
do similar for the uWSGI infrastructure (with less comfort), the folders are::
/etc/uwsgi/apps-available/
/etc/uwsgi/apps-enabled/
The `uwsgi ini file`_ is enabled by a symbolic link::
ln -s /etc/uwsgi/apps-available/searx.ini /etc/uwsgi/apps-enabled/
From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You
could control specific instance(s) by issuing::
service uwsgi <command> <confname> <confname> ...
sudo -H service uwsgi start searx
sudo -H service uwsgi stop searx
My experience is, that this command is a bit buggy.
.. _uwsgi configuration:
Alltogether
===========
Create the configuration ini-file according to your distribution (see below) and
restart the uwsgi application.
.. tabs::
.. group-tab:: Ubuntu / debian
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START searx uwsgi-description ubuntu-20.04
:end-before: END searx uwsgi-description ubuntu-20.04
.. group-tab:: Arch Linux
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START searx uwsgi-description arch
:end-before: END searx uwsgi-description arch
.. group-tab:: Fedora / RHEL
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START searx uwsgi-description fedora
:end-before: END searx uwsgi-description fedora
.. tabs::
.. group-tab:: Ubuntu / debian
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:code: ini
:start-after: START searx uwsgi-appini ubuntu-20.04
:end-before: END searx uwsgi-appini ubuntu-20.04
.. group-tab:: Arch Linux
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:code: ini
:start-after: START searx uwsgi-appini arch
:end-before: END searx uwsgi-appini arch
.. group-tab:: Fedora / RHEL
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:code: ini
:start-after: START searx uwsgi-appini fedora
:end-before: END searx uwsgi-appini fedora

357
docs/admin/installation.rst
View File

@ -4,346 +4,63 @@
Installation
============
.. contents::
:depth: 3
*You're spoilt for choice*, choose your preferred method of installation.
Basic installation
==================
- :ref:`installation docker`
- :ref:`installation scripts`
- :ref:`installation basic`
Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure
to have enable universe repository.
The :ref:`installation basic` is good enough for intranet usage and it is a
excellent illustration of *how a searx instance is build up*. If you place your
instance public to the internet you should really consider to install a
:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy
<morty.sh>` is mandatory.
Install packages:
Therefore, if you do not have any special preferences, its recommend to use the
:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling
box <toolboxing>` as described below.
.. code:: sh
.. _installation scripts:
$ sudo -H apt-get install \
git build-essential libxslt-dev \
python-dev python-virtualenv python-babel \
zlib1g-dev libffi-dev libssl-dev
Installation scripts
====================
Install searx:
.. sidebar:: Update OS first!
.. code:: sh
To avoid unwanted side effects, update your OS before installing searx.
cd /usr/local
sudo -H git clone https://github.com/asciimoo/searx.git
sudo -H useradd searx -d /usr/local/searx
sudo -H chown searx:searx -R /usr/local/searx
The following will install a setup as shown in :ref:`architecture`. First you
need to get a clone. The clone is only needed for the installation procedure
and some maintenance tasks (alternatively you can create your own fork).
Install dependencies in a virtualenv:
.. code:: bash
.. code:: sh
$ cd ~/Downloads
$ git clone https://github.com/asciimoo/searx searx
$ cd searx
cd /usr/local/searx
sudo -H -u searx -i
**Install** :ref:`searx service <searx.sh>`
.. code:: sh
This installs searx as described in :ref:`installation basic`.
(searx)$ virtualenv searx-ve
(searx)$ . ./searx-ve/bin/activate
(searx)$ ./manage.sh update_packages
.. code:: bash
Configuration
==============
$ sudo -H ./utils/searx.sh install all
.. code:: sh
**Install** :ref:`filtron reverse proxy <filtron.sh>`
sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml
.. code:: bash
Edit searx/settings.yml if necessary.
$ sudo -H ./utils/filtron.sh install all
Check
=====
**Install** :ref:`result proxy <morty.sh>`
Start searx:
.. code:: bash
.. code:: sh
$ sudo -H ./utils/morty.sh install all
python searx/webapp.py
If all services are running fine, you can add it to your HTTP server:
Go to http://localhost:8888
- :ref:`installation apache`
- :ref:`installation nginx`
If everything works fine, disable the debug option in settings.yml:
.. code:: sh
sed -i -e "s/debug : True/debug : False/g" searx/settings.yml
At this point searx is not demonized ; uwsgi allows this.
You can exit the virtualenv and the searx user bash (enter exit command
twice).
uwsgi
=====
Install packages:
.. code:: sh
sudo -H apt-get install \
uwsgi uwsgi-plugin-python
Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this
content:
.. code:: ini
[uwsgi]
# Who will run the code
uid = searx
gid = searx
# disable logging for privacy
disable-logging = true
# Number of workers (usually CPU count)
workers = 4
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
master = true
plugin = python
lazy-apps = true
enable-threads = true
# Module to import
module = searx.webapp
# Support running the module from a webserver subdirectory.
route-run = fixpathinfo:
# Virtualenv and python path
virtualenv = /usr/local/searx/searx-ve/
pythonpath = /usr/local/searx/
chdir = /usr/local/searx/searx/
Activate the uwsgi application and restart:
.. code:: sh
cd /etc/uwsgi/apps-enabled
ln -s ../apps-available/searx.ini
/etc/init.d/uwsgi restart
Web server
==========
with nginx
----------
If nginx is not installed (uwsgi will not work with the package
nginx-light):
.. code:: sh
sudo -H apt-get install nginx
Hosted at /
~~~~~~~~~~~
Create the configuration file ``/etc/nginx/sites-available/searx`` with this
content:
.. code:: nginx
server {
listen 80;
server_name searx.example.com;
root /usr/local/searx/searx;
location /static {
}
location / {
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
}
Create a symlink to sites-enabled:
.. code:: sh
sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
Restart service:
.. code:: sh
sudo -H service nginx restart
sudo -H service uwsgi restart
from subdirectory URL (/searx)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Add this configuration in the server config file
``/etc/nginx/sites-enabled/default``:
.. code:: nginx
location /searx/static {
alias /usr/local/searx/searx/static;
}
location /searx {
uwsgi_param SCRIPT_NAME /searx;
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
**OR** using reverse proxy (Please, note that reverse proxy advised to be used
in case of single-user or low-traffic instances.)
.. code:: nginx
location /searx/static {
alias /usr/local/searx/searx/static;
}
location /searx {
proxy_pass http://127.0.0.1:8888;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
proxy_buffering off;
}
Enable ``base_url`` in ``searx/settings.yml``
.. code:: yaml
base_url : http://your.domain.tld/searx/
Restart service:
.. code:: sh
sudo -H service nginx restart
sudo -H service uwsgi restart
disable logs
^^^^^^^^^^^^
for better privacy you can disable nginx logs about searx.
how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default``
add:
.. code:: nginx
access_log /dev/null;
error_log /dev/null;
Restart service:
.. code:: sh
sudo -H service nginx restart
with apache
-----------
Add wsgi mod:
.. code:: sh
sudo -H apt-get install libapache2-mod-uwsgi
sudo -H a2enmod uwsgi
Add this configuration in the file ``/etc/apache2/apache2.conf``:
.. code:: apache
<Location />
Options FollowSymLinks Indexes
SetHandler uwsgi-handler
uWSGISocket /run/uwsgi/app/searx/socket
</Location>
Note that if your instance of searx is not at the root, you should change
``<Location />`` by the location of your instance, like ``<Location /searx>``.
Restart Apache:
.. code:: sh
sudo -H /etc/init.d/apache2 restart
disable logs
~~~~~~~~~~~~
For better privacy you can disable Apache logs.
.. warning::
You can only disable logs for the whole (virtual) server not for a specific
path.
Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add:
.. code:: apache
CustomLog /dev/null combined
Restart Apache:
.. code:: sh
sudo -H /etc/init.d/apache2 restart
How to update
=============
.. code:: sh
cd /usr/local/searx
sudo -H -u searx -i
.. code:: sh
(searx)$ . ./searx-ve/bin/activate
(searx)$ git stash
(searx)$ git pull origin master
(searx)$ git stash apply
(searx)$ ./manage.sh update_packages
.. code:: sh
sudo -H service uwsgi restart
Docker
======
Make sure you have installed Docker. For instance, you can deploy searx like this:
.. code:: sh
docker pull wonderfall/searx
docker run -d --name searx -p $PORT:8888 wonderfall/searx
Go to ``http://localhost:$PORT``.
See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also
possible to build searx from the embedded Dockerfile.
.. code:: sh
git clone https://github.com/asciimoo/searx.git
cd searx
docker build -t whatever/searx .
References
==========
* https://about.okhin.fr/posts/Searx/ with some additions
* How to: `Setup searx in a couple of hours with a free SSL certificate
<https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__

7
docs/admin/morty.rst
View File

@ -1,7 +1,14 @@
.. _searx morty:
=========================
How to setup result proxy
=========================
.. sidebar:: further reading
- :ref:`morty.sh`
.. _morty: https://github.com/asciimoo/morty
.. _morty's README: https://github.com/asciimoo/morty

8
docs/admin/settings.rst
View File

@ -4,11 +4,17 @@
``settings.yml``
================
This page describe the options possibilities of the :origin:`searx/settings.yml`
file.
.. sidebar:: Further reading ..
- :ref:`search API`
This page describe the options possibilities of the settings.yml file.
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
.. _settings global:

23
docs/admin/update-searx.rst Normal file
View File

@ -0,0 +1,23 @@
.. _update searx:
=============
How to update
=============
.. code:: sh
sudo -H -u searx -i
(searx)$ git stash
(searx)$ git pull origin master
(searx)$ git stash apply
(searx)$ ./manage.sh update_packages
Restart uwsgi:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H systemctl restart uwsgi

3
docs/blog/index.rst
View File

@ -3,7 +3,8 @@ Blog
====
.. toctree::
:maxdepth: 1
:maxdepth: 2
:caption: Contents
python3
admin

52
docs/build-templates/filtron.rst Normal file
View File

@ -0,0 +1,52 @@
.. START create user
.. tabs::
.. group-tab:: bash
.. code-block:: sh
$ sudo -H useradd --shell /bin/bash --system \\
--home-dir "$SERVICE_HOME" \\
--comment "Privacy-respecting metasearch engine" $SERVICE_USER
$ sudo -H mkdir "$SERVICE_HOME"
$ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
.. END create user
.. START install go
.. tabs::
.. group-tab:: bash
.. code-block:: bash
$ cat > "$GO_ENV" <<EOF
export GOPATH=${SERVICE_HOME}/go-apps
export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
EOF
$ sudo -i -u "${SERVICE_USER}"
(${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
(${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
(${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\
"${GO_PKG_URL}"
(${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}"
(${SERVICE_USER}) $ which go
${SERVICE_HOME}/local/go/bin/go
.. END install go
.. START install filtron
.. tabs::
.. group-tab:: bash
.. code-block:: bash
$ sudo -i -u "${SERVICE_USER}"
(${SERVICE_USER}) $ go get -v -u github.com/asciimoo/filtron
.. END install filtron

52
docs/build-templates/morty.rst Normal file
View File

@ -0,0 +1,52 @@
.. START create user
.. tabs::
.. group-tab:: bash
.. code-block:: sh
$ sudo -H useradd --shell /bin/bash --system \\
--home-dir "$SERVICE_HOME" \\
--comment "Privacy-respecting metasearch engine" $SERVICE_USER
$ sudo -H mkdir "$SERVICE_HOME"
$ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
.. END create user
.. START install go
.. tabs::
.. group-tab:: bash
.. code-block:: bash
$ cat > "$GO_ENV" <<EOF
export GOPATH=${SERVICE_HOME}/go-apps
export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin
EOF
$ sudo -i -u "${SERVICE_USER}"
(${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile
(${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
(${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\
"${GO_PKG_URL}"
(${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}"
(${SERVICE_USER}) $ which go
${SERVICE_HOME}/local/go/bin/go
.. END install go
.. START install morty
.. tabs::
.. group-tab:: bash
.. code-block:: bash
$ sudo -i -u "${SERVICE_USER}"
(${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty
.. END install morty

192
docs/build-templates/searx.rst Normal file
View File

@ -0,0 +1,192 @@
.. template evaluated by: ./utils/searx.sh docs
.. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$
.. START distro-packages
.. tabs::
.. group-tab:: Ubuntu / debian
.. code-block:: sh
$ sudo -H apt-get install -y \\
${debian}
.. group-tab:: Arch Linux
.. code-block:: sh
$ sudo -H pacman -S --noconfirm \\
${arch}
.. group-tab:: Fedora / RHEL
.. code-block:: sh
$ sudo -H dnf install -y \\
${fedora}
.. END distro-packages
.. START build-packages
.. tabs::
.. group-tab:: Ubuntu / debian
.. code-block:: sh
$ sudo -H apt-get install -y \\
${debian_build}
.. group-tab:: Arch Linux
.. code-block:: sh
$ sudo -H pacman -S --noconfirm \\
${arch_build}
.. group-tab:: Fedora / RHEL
.. code-block:: sh
$ sudo -H dnf install -y \\
${fedora_build}
.. END build-packages
.. START create user
.. tabs::
.. group-tab:: bash
.. code-block:: sh
$ sudo -H useradd --shell /bin/bash --system \\
--home-dir "$SERVICE_HOME" \\
--comment "Privacy-respecting metasearch engine" $SERVICE_USER
$ sudo -H mkdir "$SERVICE_HOME"
$ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
.. END create user
.. START clone searx
.. tabs::
.. group-tab:: bash
.. code-block:: sh
$ sudo -H -u ${SERVICE_USER} -i
(${SERVICE_USER})$ git clone "https://github.com/asciimoo/searx.git" "$SEARX_SRC"
.. END clone searx
.. START create virtualenv
.. tabs::
.. group-tab:: bash
.. code-block:: sh
(${SERVICE_USER})$ python3 -m venv "${SEARX_PYENV}"
(${SERVICE_USER})$ echo ". ${SEARX_PYENV}/bin/activate" >> "$SERVICE_HOME/.profile"
.. END create virtualenv
.. START manage.sh update_packages
.. tabs::
.. group-tab:: bash
.. code-block:: sh
$ sudo -H -u ${SERVICE_USER} -i
(${SERVICE_USER})$ command -v python && python --version
$SEARX_PYENV/bin/python
Python 3.8.1
# update pip's boilerplate ..
pip install -U pip
pip install -U setuptools
pip install -U wheel
# jump to searx's working tree and install searx into virtualenv
(${SERVICE_USER})$ cd "$SEARX_SRC"
(${SERVICE_USER})$ pip install -e .
.. END manage.sh update_packages
.. START searx config
.. tabs::
.. group-tab:: bash
.. code-block:: sh
$ sudo -H cp "$SEARX_SRC/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
$ sudo -H sed -i -e "s/ultrasecretkey/\\$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
$ sudo -H sed -i -e "s/{instance_name}/searx@\\$(uname -n)/g" "$SEARX_SETTINGS_PATH"
.. END searx config
.. START check searx installation
.. tabs::
.. group-tab:: bash
.. code-block:: sh
# enable debug ..
$ sudo -H sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
# start webapp
$ sudo -H -u ${SERVICE_USER} -i
(${SERVICE_USER})$ cd ${SEARX_SRC}
(${SERVICE_USER})$ export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}"
(${SERVICE_USER})$ python searx/webapp.py
# disable debug
$ sudo -H sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
Open WEB browser and visit http://$SEARX_INTERNAL_URL . If you are inside a
container or in a script, test with curl:
.. tabs::
.. group-tab:: WEB browser
.. code-block:: sh
$ xgd-open http://$SEARX_INTERNAL_URL
.. group-tab:: curl
.. code-block:: none
$ curl --location --verbose --head --insecure $SEARX_INTERNAL_URL
* Trying 127.0.0.1:8888...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8888 (#0)
> HEAD / HTTP/1.1
> Host: 127.0.0.1:8888
> User-Agent: curl/7.68.0
> Accept: */*
>
* Mark bundle as not supporting multiuse
* HTTP 1.0, assume close after body
< HTTP/1.0 200 OK
HTTP/1.0 200 OK
...
.. END check searx installation

14
docs/conf.py
View File

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-
import sys, os
from sphinx_build_tools import load_sphinx_config
from searx.version import VERSION_STRING
from pallets_sphinx_themes import ProjectLink
from searx.brand import GIT_URL
GIT_BRANCH = os.environ.get("GIT_BRANCH", "master")
from searx.brand import SEARX_URL
from searx.brand import DOCS_URL
@ -22,6 +24,8 @@ master_doc = "index"
source_suffix = '.rst'
numfig = True
exclude_patterns = ['build-templates/*.rst']
from searx import webapp
jinja_contexts = {
'webapp': dict(**webapp.__dict__)
@ -35,7 +39,7 @@ extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ')
extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ')
# links to custom brand
extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://')
extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
extlinks['search'] = (SEARX_URL + '/%s', '#')
extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
@ -61,6 +65,8 @@ extensions = [
"pallets_sphinx_themes",
"sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
"sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja
"sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput
'linuxdoc.kernel_include', # Implementation of the 'kernel-include' reST-directive.
'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive.
'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling.
"sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
@ -112,3 +118,9 @@ html_show_sourcelink = False
latex_documents = [
(master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
]
# ------------------------------------------------------------------------------
# Since loadConfig overwrites settings from the global namespace, it has to be
# the last statement in the conf.py file
# ------------------------------------------------------------------------------
load_sphinx_config(globals())

5
docs/dev/contribution_guide.rst
View File

@ -4,6 +4,11 @@
How to contribute
=================
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
Prime directives: Privacy, Hackability
======================================

3
docs/dev/index.rst
View File

@ -3,7 +3,8 @@ Developer documentation
=======================
.. toctree::
:maxdepth: 1
:maxdepth: 2
:caption: Contents
quickstart
contribution_guide

46
docs/dev/makefile.rst
View File

@ -11,23 +11,17 @@ Makefile Targets
Before looking deeper at the targets, first read about :ref:`makefile setup`
and :ref:`make pyenv`.
To install system requirements follow :ref:`buildhosts`.
With the aim to simplify development cycles, started with :pull:`1756` a
``Makefile`` based boilerplate was added. If you are not familiar with
Makefiles, we recommend to read gnu-make_ introduction.
The usage is simple, just type ``make {target-name}`` to *build* a target.
Calling the ``help`` target gives a first overview::
Calling the ``help`` target gives a first overview (``make help``):
.. program-output:: bash -c "cd ..; make --no-print-directory help"
$ make help
test - run developer tests
docs - build documentation
docs-live - autobuild HTML documentation while editing
run - run developer instance
install - developer install (./local)
uninstall - uninstall (./local)
gh-pages - build docs & deploy on gh-pages branch
clean - drop builds and environments
...
.. contents:: Contents
:depth: 2
@ -37,27 +31,33 @@ Calling the ``help`` target gives a first overview::
.. _makefile setup:
Setup
=====
Makefile setup
==============
.. _git stash: https://git-scm.com/docs/git-stash
The main setup is done in the :origin:`Makefile`::
export GIT_URL=https://github.com/asciimoo/searx
export SEARX_URL=https://searx.me
export DOCS_URL=https://asciimoo.github.io/searx
.. sidebar:: fork & upstream
Commit changes in your (local) branch, fork or whatever, but do not push them
upstream / `git stash`_ is your friend.
:GIT_URL: Changes this, to point to your searx fork.
The main setup is done in the :origin:`Makefile`.
:SEARX_URL: Changes this, to point to your searx instance.
.. literalinclude:: ../../Makefile
:start-after: START Makefile setup
:end-before: END Makefile setup
:DOCS_URL: If you host your own (branded) documentation, change this URL.
:GIT_URL: Changes this, to point to your searx fork.
:GIT_BRANCH: Changes this, to point to your searx branch.
:SEARX_URL: Changes this, to point to your searx instance.
:DOCS_URL: If you host your own (*brand*) documentation, change this URL.
If you change any of this build environment variables, you have to run ``make
buildenv``::
$ make buildenv
build searx/brand.py
build utils/brand.env
.. _make pyenv:
@ -170,7 +170,7 @@ e.g.:
.. code:: sh
$ make test.pep8 test.unit
$ make test.pep8 test.unit test.sh
. ./local/py3/bin/activate; ./manage.sh pep8_check
[!] Running pep8 check
. ./local/py3/bin/activate; ./manage.sh unit_tests

2
docs/dev/quickstart.rst
View File

@ -27,7 +27,7 @@ searx-ve virtualenv and install the required packages using ``manage.sh``.
cd ~/myprojects
git clone https://github.com/asciimoo/searx.git
cd searx
virtualenv searx-ve
python3 -m venv searx-ve
. ./searx-ve/bin/activate
./manage.sh update_dev_packages

10
docs/dev/reST.rst
View File

@ -325,8 +325,9 @@ Literal blocks
The simplest form of :duref:`literal-blocks` is a indented block introduced by
two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST
code` directive. To include literals from external files use directive
:dudir:`literalinclude`.
code` directive. To include literals from external files use
:rst:dir:`literalinclude` or :ref:`kernel-include <kernel-include-directive>`
directive (latter one expands environment variables in the path name).
.. _reST literal:
@ -1312,9 +1313,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from
.. literalinclude:: ../admin/buildhosts.rst
:language: reST
:start-after: .. _system requirements:
:end-before: .. _system requirements END:
:start-after: .. SNIP sh lint requirements
:end-before: .. SNAP sh lint requirements
.. _math:

18
docs/index.rst
View File

@ -2,7 +2,14 @@
Welcome to searx
================
Search without being tracked.
*Search without being tracked.*
Searx is a free internet metasearch engine which aggregates results from more
than 70 search services. Users are neither tracked nor profiled. Additionally,
searx can be used over Tor for online anonymity.
Get started with searx by using one of the Searx-instances_. If you don't trust
anyone, you can set up your own, see :ref:`installation`.
.. sidebar:: Features
@ -16,19 +23,14 @@ Search without being tracked.
- Hosted by organizations, such as *La Quadrature du Net*, which promote
digital rights
Searx is a free internet metasearch engine which aggregates results from more
than 70 search services. Users are neither tracked nor profiled. Additionally,
searx can be used over Tor for online anonymity.
Get started with searx by using one of the Searx-instances_. If you don't trust
anyone, you can set up your own, see :ref:`installation`.
.. toctree::
:maxdepth: 2
:caption: Contents
user/index
admin/index
dev/index
utils/index
blog/index
.. _Searx-instances: https://searx.space

19
docs/user/conf.py Normal file
View File

@ -0,0 +1,19 @@
# -*- coding: utf-8; mode: python -*-
"""Configuration for the Searx user handbook
"""
project = 'Searx User-HB'
version = release = VERSION_STRING
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index' # startdocname
, 'searx-user-hb.tex' # targetname
, '' # take title from .rst
, author # author
, 'howto' # documentclass
, False # toctree_only
),
]

3
docs/user/index.rst
View File

@ -3,7 +3,8 @@ User documentation
==================
.. toctree::
:maxdepth: 1
:maxdepth: 2
:caption: Contents
search_syntax
own-instance

6
docs/user/own-instance.rst
View File

@ -2,8 +2,10 @@
Why use a private instance?
===========================
"Is it worth to run my own instance?" is a common question among searx users.
Before answering this question, see what options a searx user has.
*"Is it worth to run my own instance?"*
\.\. is a common question among searx users. Before answering this question,
see what options a searx user has.
Public instances are open to everyone who has access to its URL. Usually, these
are operated by unknown parties (from the users' point of view). Private

3
docs/user/public_instances.rst
View File

@ -1,3 +0,0 @@
:orphan:
This page page has been moved to `searx.space <https://searx.space/>`__

80
docs/utils/filtron.sh.rst Normal file
View File

@ -0,0 +1,80 @@
.. _filtron.sh:
====================
``utils/filtron.sh``
====================
.. sidebar:: further reading
- :ref:`searx filtron`
- :ref:`architecture`
- :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
<installation apache>`)
.. _Go: https://golang.org/
.. _filtron: https://github.com/asciimoo/filtron
.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md
To simplify installation and maintenance of a filtron instance you can use the
script :origin:`utils/filtron.sh`. In most cases you will install filtron_
simply by running the command:
.. code:: bash
sudo -H ./utils/filtron.sh install all
The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_
into this user account:
#. Create a separated user account (``filtron``).
#. Download and install Go_ binary in user's $HOME (``~filtron``).
#. Install filtron with the package management from Go_ (``go get -v -u
github.com/asciimoo/filtron``)
#. Setup a proper rule configuration :origin:`[ref]
<utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``).
#. Setup a systemd service unit :origin:`[ref]
<utils/templates/lib/systemd/system/filtron.service>`
(``/lib/systemd/system/filtron.service``).
Create user
===========
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START create user
:end-before: END create user
Install go
==========
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START install go
:end-before: END install go
Install filtron
===============
Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at
``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and
install filtron software and systemd unit:
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START install filtron
:end-before: END install filtron
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START install systemd unit
:end-before: END install systemd unit
.. _filtron.sh overview:
Overview
========
The ``--help`` output of the script is largely self-explanatory
(:ref:`toolboxing common`):
.. program-output:: ../utils/filtron.sh --help

53
docs/utils/index.rst Normal file
View File

@ -0,0 +1,53 @@
.. _searx_utils:
.. _toolboxing:
=======================
Tooling box ``utils/*``
=======================
In the folder :origin:`utils/` we maintain some tools useful for admins and
developers.
.. toctree::
:maxdepth: 2
:caption: Contents
searx.sh
filtron.sh
morty.sh
lxc.sh
.. _toolboxing common:
Common commands & environment
=============================
Scripts to maintain services often dispose of common commands and environments.
``shell`` : command
Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
troubleshooting.
``inspect service`` : command
Shows status and log of the service, most often you have a option to enable
more verbose debug logs. Very helpful for debugging, but be careful not to
enable debugging in a production environment!
``FORCE_TIMEOUT`` : environment
Sets timeout for interactive prompts. If you want to run a script in batch
job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
reverse proxy for filtron on all containers of the :ref:`searx suite
<lxc-searx.env>` use ::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
.. _toolboxing setup:
Tooling box setup
=================
The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile
setup`).
.. literalinclude:: ../../.config.sh
:language: bash

148
docs/utils/lxc.sh.rst Normal file
View File

@ -0,0 +1,148 @@
.. _snap: https://snapcraft.io
.. _snapcraft LXD: https://snapcraft.io/lxd
.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/
.. _LXC: https://linuxcontainers.org/lxc/introduction/
.. _LXD: https://linuxcontainers.org/lxd/introduction/
.. _`LXD@github`: https://github.com/lxc/lxd
.. _archlinux: https://www.archlinux.org/
.. _lxc.sh:
================
``utils/lxc.sh``
================
.. sidebar:: further reading
- snap_, `snapcraft LXD`_
- LXC_, LXD_
- `LXC/LXD Image Server`_
- `LXD@github`_
With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of
containers, what we call the: *lxc suite*. The *searx suite*
(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time
you start the ``lxc.sh`` script (*you do not need to care about*).
Before you can start with containers, you need to install and initiate LXD_
once::
$ snap install lxd
$ lxd init --auto
To make use of the containers from the *searx suite*, you have to build the
:ref:`LXC suite containers <lxc.sh help>` initial. But be warned, **this might
take some time**::
$ sudo -H ./utils/lxc.sh build
A cup of coffee later, your LXC suite is build up and you can run whatever task
you want / in a selected or even in all :ref:`LXC suite containers <lxc.sh
help>`. If you do not want to build all containers, **you can build just
one**::
$ sudo -H ./utils/lxc.sh build searx-ubu1804
*Good to know ...*
Eeach container shares the root folder of the repository and the
command ``utils/lxc.sh cmd`` **handles relative path names transparent**,
compare output of::
$ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile
...
In the containers, you can run what ever you want, e.g. to start a bash use::
$ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash
INFO: [searx-ubu1804] bash
root@searx-ubu1804:/share/searx#
If there comes the time you want to **get rid off all** the containers and
**clean up local images** just type::
$ sudo -H ./utils/lxc.sh remove
$ sudo -H ./utils/lxc.sh remove images
Install suite
=============
To install the complete :ref:`searx suite (includes searx, morty & filtron)
<lxc-searx.env>` into all LXC_ use::
$ sudo -H ./utils/lxc.sh install suite
The command above installs a searx suite (see :ref:`installation scripts`). To
get the IP (URL) of the filtron service in the containers use ``show suite``
command. To test instances from containers just open the URLs in your
WEB-Browser::
$ sudo ./utils/lxc.sh show suite | grep filtron
[searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx
[searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx
[searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx
[searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx
[searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx
[searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx
To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and
morty use (or alternatively use :ref:`apache <installation apache>`)::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install
Running commands
================
**Inside containers, you can use make or run scripts** from the
:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the
Makefile target ``test`` in the archlinux_ container::
sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost
sudo -H ./utils/lxc.sh cmd searx-archlinux make test
Setup searx buildhost
=====================
You can **install the searx buildhost environment** into one or all containers.
The installation procedure to set up a :ref:`build host<buildhosts>` takes its
time. Installation in all containers will take more time (time for another cup
of coffee).::
sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost
To build (live) documentation inside a archlinux_ container::
sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live
...
[I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
To get IP of the container and the port number *live docs* is listening::
$ sudo ./utils/lxc.sh show suite | grep docs-live
...
[searx-archlinux] INFO: (eth0) docs-live: http://n.n.n.12:8080/
.. _lxc.sh help:
Overview
========
The ``--help`` output of the script is largely self-explanatory:
.. program-output:: ../utils/lxc.sh --help
.. _lxc-searx.env:
searx suite
===========
.. literalinclude:: ../../utils/lxc-searx.env
:language: bash

80
docs/utils/morty.sh.rst Normal file
View File

@ -0,0 +1,80 @@
.. _morty: https://github.com/asciimoo/morty
.. _morty's README: https://github.com/asciimoo/morty
.. _Go: https://golang.org/
.. _morty.sh:
==================
``utils/morty.sh``
==================
.. sidebar:: further reading
- :ref:`architecture`
- :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
<installation apache>`)
- :ref:`searx morty`
To simplify installation and maintenance of a morty_ instance you can use the
script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by
running the command:
.. code:: bash
sudo -H ./utils/morty.sh install all
The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_
into this user account:
#. Create a separated user account (``morty``).
#. Download and install Go_ binary in user's $HOME (``~morty``).
#. Install morty_ with the package management from Go_ (``go get -v -u
github.com/asciimoo/morty``)
#. Setup a systemd service unit :origin:`[ref]
<utils/templates/lib/systemd/system/morty.service>`
(``/lib/systemd/system/morty.service``).
.. hint::
To add morty to your searx instance read chapter :ref:`searx morty`.
Create user
===========
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START create user
:end-before: END create user
Install go
==========
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START install go
:end-before: END install go
Install morty
=============
Install morty software and systemd unit:
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START install morty
:end-before: END install morty
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START install systemd unit
:end-before: END install systemd unit
.. _morty.sh overview:
Overview
========
The ``--help`` output of the script is largely self-explanatory
(:ref:`toolboxing common`):
.. program-output:: ../utils/morty.sh --help

39
docs/utils/searx.sh.rst Normal file
View File

@ -0,0 +1,39 @@
.. _searx.sh:
==================
``utils/searx.sh``
==================
.. sidebar:: further reading
- :ref:`architecture`
- :ref:`installation`
- :ref:`installation nginx`
- :ref:`installation apache`
To simplify installation and maintenance of a searx instance you can use the
script :origin:`utils/searx.sh`.
Install
=======
In most cases you will install searx simply by running the command:
.. code:: bash
sudo -H ./utils/searx.sh install all
The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs searx
into this user account. The installation is described in chapter
:ref:`installation basic`.
.. _intranet reverse proxy:
Overview
========
The ``--help`` output of the script is largely self-explanatory
(:ref:`toolboxing common`):
.. program-output:: ../utils/searx.sh --help

2
requirements-dev.txt
View File

@ -15,3 +15,5 @@ selenium==3.141.0
linuxdoc @ git+http://github.com/return42/linuxdoc.git
sphinx-jinja
sphinx-tabs
sphinxcontrib-programoutput
twine

1
searx/brand.py
View File

@ -1,4 +1,5 @@
GIT_URL = 'https://github.com/asciimoo/searx'
GIT_BRANCH = 'master'
ISSUE_URL = 'https://github.com/asciimoo/searx/issues'
SEARX_URL = 'https://searx.me'
DOCS_URL = 'https://asciimoo.github.io/searx'

1
utils/brand.env
View File

@ -1,4 +1,5 @@
export GIT_URL='https://github.com/asciimoo/searx'
export GIT_BRANCH='master'
export ISSUE_URL='https://github.com/asciimoo/searx/issues'
export SEARX_URL='https://searx.me'
export DOCS_URL='https://asciimoo.github.io/searx'

561
utils/filtron.sh Executable file
View File

@ -0,0 +1,561 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2119,SC2001
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
source_dot_config
source "${REPO_ROOT}/utils/lxc-searx.env"
in_container && lxc_set_suite_env
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
FILTRON_ETC="/etc/filtron"
FILTRON_RULES="$FILTRON_ETC/rules.json"
FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}"
SERVICE_NAME="filtron"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
GO_ENV="${SERVICE_HOME}/.go_env"
GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
GO_TAR=$(basename "$GO_PKG_URL")
APACHE_FILTRON_SITE="searx.conf"
NGINX_FILTRON_SITE="searx.conf"
# shellcheck disable=SC2034
CONFIG_FILES=(
"${FILTRON_RULES}"
"${SERVICE_SYSTEMD_UNIT}"
)
# ----------------------------------------------------------------------------
usage() {
# ----------------------------------------------------------------------------
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|user|rules]
$(basename "$0") update [filtron]
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off]
$(basename "$0") apache [install|remove]
$(basename "$0") nginx [install|remove]
shell
start interactive shell from user ${SERVICE_USER}
install / remove
:all: complete setup of filtron service
:user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
:rules: reinstall filtron rules $FILTRON_RULES
update filtron
Update filtron installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
show service status and log
option
set one of the available options
apache (${PUBLIC_URL})
:install: apache site with a reverse proxy (ProxyPass)
:remove: apache site ${APACHE_FILTRON_SITE}
nginx (${PUBLIC_URL})
:install: nginx site with a reverse proxy (ProxyPass)
:remove: nginx site ${NGINX_FILTRON_SITE}
filtron rules: ${FILTRON_RULES}
If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
PUBLIC_URL : ${PUBLIC_URL}
PUBLIC_HOST : ${PUBLIC_HOST}
SERVICE_USER : ${SERVICE_USER}
FILTRON_TARGET : ${FILTRON_TARGET}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
EOF
if in_container; then
# in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container URL (IPv6): http://[${ip#*|}]:4005/"
else
# IPv4:
echo " container URL (IPv4): http://${ip#*|}:4005/"
fi
done
fi
[[ -n ${1} ]] && err_msg "$1"
}
main() {
required_commands \
sudo install git wget curl \
|| exit
local _usage="unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SERVICE_NAME" part
sudo_or_exit
case $2 in
all) install_all ;;
user) assert_user ;;
rules)
rst_title "Re-Install filtron rules"
echo
install_template --no-eval "$FILTRON_RULES" root root 644
systemd_restart_service "${SERVICE_NAME}"
;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
filtron) update_filtron ;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service) systemd_activate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) systemd_deactivate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
install) install_nginx_site ;;
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
debug-on) echo; enable_debug ;;
debug-off) echo; disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc ;;
*) usage "unknown or missing command $1"; exit 42;;
esac
}
install_all() {
rst_title "Install $SERVICE_NAME (service)"
assert_user
wait_key
install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
wait_key
install_filtron
wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
wait_key
echo
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
elif nginx_is_installed; then
info_msg "nginx is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_nginx_site
fi
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
rst_para "\
It goes without saying that this script can only be used to remove
installations that were installed with this script."
if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
return 42
fi
drop_service_account "${SERVICE_USER}"
rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
if service_is_available "${PUBLIC_URL}"; then
MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
export SERVICE_HOME
echo "export SERVICE_HOME=$SERVICE_HOME"
cat > "$GO_ENV" <<EOF
export GOPATH=\$HOME/go-apps
export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
EOF
echo "Environment $GO_ENV has been setup."
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
EOF
}
filtron_is_installed() {
[[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
}
_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
install_filtron() {
rst_title "Install filtron in user's ~/go-apps" section
echo
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
go get -v -u github.com/asciimoo/filtron
EOF
install_template --no-eval "$FILTRON_RULES" root root 644
}
update_filtron() {
rst_title "Update filtron" section
echo
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
go get -v -u github.com/asciimoo/filtron
EOF
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
PUBLIC_URL : ${PUBLIC_URL}
PUBLIC_HOST : ${PUBLIC_HOST}
FILTRON_URL_PATH : ${FILTRON_URL_PATH}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_TARGET : ${FILTRON_TARGET}
EOF
if service_account_is_available "$SERVICE_USER"; then
info_msg "service account $SERVICE_USER available."
else
err_msg "service account $SERVICE_USER not available!"
fi
if go_is_available "$SERVICE_USER"; then
info_msg "~$SERVICE_USER: go is installed"
else
err_msg "~$SERVICE_USER: go is not installed"
fi
if filtron_is_installed; then
info_msg "~$SERVICE_USER: filtron app is installed"
else
err_msg "~$SERVICE_USER: filtron app is not installed!"
fi
if ! service_is_available "http://${FILTRON_API}"; then
err_msg "API not available at: http://${FILTRON_API}"
fi
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
fi
if service_is_available "http://${FILTRON_TARGET}" ; then
info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
fi
if ! service_is_available "${PUBLIC_URL}"; then
warn_msg "Public service at ${PUBLIC_URL} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL}"
info_msg "internal URL --> http://${FILTRON_LISTEN}"
fi
local _debug_on
if ask_yn "Enable filtron debug mode?"; then
enable_debug
_debug_on=1
fi
echo
systemctl --no-pager -l status "${SERVICE_NAME}"
echo
info_msg "public URL --> ${PUBLIC_URL}"
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
journalctl -f -u "${SERVICE_NAME}"
done
if [[ $_debug_on == 1 ]]; then
disable_debug
fi
return 0
}
enable_debug() {
info_msg "try to enable debug mode ..."
python <<EOF
import sys, json
debug = {
u'name': u'debug request'
, u'filters': []
, u'interval': 0
, u'limit': 0
, u'actions': [{u'name': u'log'}]
}
with open('$FILTRON_RULES') as rules:
j = json.load(rules)
pos = None
for i in range(len(j)):
if j[i].get('name') == 'debug request':
pos = i
break
if pos is not None:
j[pos] = debug
else:
j.append(debug)
with open('$FILTRON_RULES', 'w') as rules:
json.dump(j, rules, indent=2, sort_keys=True)
EOF
systemctl restart "${SERVICE_NAME}.service"
}
disable_debug() {
info_msg "try to disable debug mode ..."
python <<EOF
import sys, json
with open('$FILTRON_RULES') as rules:
j = json.load(rules)
pos = None
for i in range(len(j)):
if j[i].get('name') == 'debug request':
pos = i
break
if pos is not None:
del j[pos]
with open('$FILTRON_RULES', 'w') as rules:
json.dump(j, rules, indent=2, sort_keys=True)
EOF
systemctl restart "${SERVICE_NAME}.service"
}
install_apache_site() {
rst_title "Install Apache site $APACHE_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
! apache_is_installed && info_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site $APACHE_FILTRON_SITE"
rst_para "\
This removes apache site ${APACHE_FILTRON_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
apache_remove_site "$APACHE_FILTRON_SITE"
}
install_nginx_site() {
rst_title "Install nginx site $NGINX_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
! nginx_is_installed && info_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_nginx
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
# shellcheck disable=SC2034
SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
# shellcheck disable=SC2034
SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_FILTRON_SITE"
rst_para "\
This removes nginx site ${NGINX_FILTRON_SITE}."
! nginx_is_installed && err_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
nginx_remove_site "$FILTRON_FILTRON_SITE"
}
rst-doc() {
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
echo -e "\n.. START install systemd unit"
cat <<EOF
.. tabs::
.. group-tab:: systemd
.. code:: bash
EOF
eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
echo -e "\n.. END install systemd unit"
# for DIST_NAME in ubuntu-20.04 arch fedora; do
# (
# DIST_ID=${DIST_NAME%-*}
# DIST_VERS=${DIST_NAME#*-}
# [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
# # ...
# )
# done
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

1519
utils/lib.sh Executable file
View File

File diff suppressed because it is too large Load Diff

95
utils/lxc-searx.env Normal file
View File

@ -0,0 +1,95 @@
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck shell=bash
# This file is a setup of a LXC suite. It is sourced from different context, do
# not manipulate the environment directly, implement functions and manipulate
# environment only is subshells!
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
# shellcheck disable=SC2034
LXC_SUITE_NAME="searx"
lxc_set_suite_env() {
# name of https://images.linuxcontainers.org
export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}"
export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}"
export LXC_SUITE=(
# to disable containers, comment out lines ..
# end of standard support see https://wiki.ubuntu.com/Releases
"$LINUXCONTAINERS_ORG_NAME:ubuntu/16.04" "ubu1604" # April 2021
"$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023
"$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" # July 2020
"$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # future (EOL 2030)
# EOL see https://fedoraproject.org/wiki/Releases
"$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31"
# rolling releases see https://www.archlinux.org/releng/releases/
"$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux"
)
PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
if in_container; then
# container hostnames do not have a DNS entry: use primary IP!
PUBLIC_URL="http://$(primary_ip)/searx"
# make GUEST's services public to the HOST
FILTRON_API="0.0.0.0:4005"
FILTRON_LISTEN="0.0.0.0:4004"
MORTY_LISTEN="0.0.0.0:3000"
# export LXC specific environment
export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN
fi
}
lxc_suite_install_info() {
(
lxc_set_suite_env
cat <<EOF
LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL}
suite includes searx, morty & filtron
suite images:
$(echo " ${LOCAL_IMAGES[*]}" | $FMT)
suite containers:
$(echo " ${CONTAINERS[*]}" | $FMT)
EOF
)
}
lxc_suite_install() {
(
lxc_set_suite_env
FORCE_TIMEOUT=0
export FORCE_TIMEOUT
"${LXC_REPO_ROOT}/utils/searx.sh" install all
"${LXC_REPO_ROOT}/utils/morty.sh" install all
"${LXC_REPO_ROOT}/utils/filtron.sh" install all
rst_title "suite installation finished ($(hostname))" part
lxc_suite_info
echo
)
}
lxc_suite_info() {
(
lxc_set_suite_env
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
info_msg "(${ip%|*}) IPv6: http://[${ip#*|}]"
else
# IPv4:
# shellcheck disable=SC2034,SC2031
info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL"
info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY"
info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/"
fi
done
)
}

552
utils/lxc.sh Executable file
View File

@ -0,0 +1,552 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
source_dot_config
# load environment of the LXC suite
LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}"
source "$LXC_ENV"
lxc_set_suite_env
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
#
# read also:
# - https://lxd.readthedocs.io/en/latest/
LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}"
# where all folders from HOST are mounted
LXC_SHARE_FOLDER="/share"
LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")"
ubu1604_boilerplate="
export DEBIAN_FRONTEND=noninteractive
apt-get update -y
apt-get upgrade -y
apt-get install -y git curl wget
"
ubu1804_boilerplate="$ubu1604_boilerplate"
ubu1904_boilerplate="$ubu1804_boilerplate"
ubu1910_boilerplate="$ubu1904_boilerplate"
# shellcheck disable=SC2034
ubu2004_boilerplate="
$ubu1910_boilerplate
echo 'Set disable_coredump false' >> /etc/sudo.conf
"
# shellcheck disable=SC2034
archlinux_boilerplate="
pacman -Syu --noconfirm
pacman -S --noconfirm inetutils git curl wget sudo
echo 'Set disable_coredump false' >> /etc/sudo.conf
"
# shellcheck disable=SC2034
fedora31_boilerplate="
dnf update -y
dnf install -y git curl wget hostname
echo 'Set disable_coredump false' >> /etc/sudo.conf
"
REMOTE_IMAGES=()
CONTAINERS=()
LOCAL_IMAGES=()
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}")
CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}")
LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}")
done
HOST_USER="${SUDO_USER:-$USER}"
HOST_USER_ID=$(id -u "${HOST_USER}")
HOST_GROUP_ID=$(id -g "${HOST_USER}")
# ----------------------------------------------------------------------------
usage() {
# ----------------------------------------------------------------------------
_cmd="$(basename "$0")"
cat <<EOF
usage::
$_cmd build [containers|<name>]
$_cmd copy [images]
$_cmd remove [containers|<name>|images]
$_cmd [start|stop] [containers|<name>]
$_cmd show [images|suite|info|config [<name>]]
$_cmd cmd [--|<name>] '...'
$_cmd install [suite|base [<name>]]
build
:containers: build, launch all containers and 'install base' packages
:<name>: build, launch container <name> and 'install base' packages
copy:
:images: copy remote images of the suite into local storage
remove
:containers: delete all 'containers' or only <container-name>
:images: delete local images of the suite
start/stop
:containers: start/stop all 'containers' from the suite
:<name>: start/stop container <name> from suite
show
:info: show info of all (or <name>) containers from LXC suite
:config: show config of all (or <name>) containers from the LXC suite
:suite: show services of all (or <name>) containers from the LXC suite
:images: show information of local images
cmd
use single qoutes to evaluate in container's bash, e.g.: 'echo \$(hostname)'
-- run command '...' in all containers of the LXC suite
:<name>: run command '...' in container <name>
install
:base: prepare LXC; install basic packages
:suite: install LXC ${LXC_SUITE_NAME} suite into all (or <name>) containers
EOF
usage_containers
[ -n "${1+x}" ] && err_msg "$1"
}
usage_containers() {
lxc_suite_install_info
[ -n "${1+x}" ] && err_msg "$1"
}
lxd_info() {
cat <<EOF
LXD is needed, to install run::
snap install lxd
lxd init --auto
EOF
}
main() {
local exit_val
local _usage="unknown or missing $1 command $2"
# don't check prerequisite when in recursion
if [[ ! $1 == __* ]]; then
if ! in_container; then
! required_commands lxc && lxd_info && exit 42
fi
[[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42
fi
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
build)
sudo_or_exit
case $2 in
${LXC_HOST_PREFIX}-*) build_container "$2" ;;
''|--|containers) build_all_containers ;;
*) usage "$_usage"; exit 42;;
esac
;;
copy)
case $2 in
''|images) lxc_copy_images_localy;;
*) usage "$_usage"; exit 42;;
esac
;;
remove)
sudo_or_exit
case $2 in
''|--|containers) remove_containers ;;
images) lxc_delete_images_localy ;;
${LXC_HOST_PREFIX}-*)
! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
if ask_yn "Do you really want to delete container $2"; then
lxc_delete_container "$2"
fi
;;
*) usage "uknown or missing container <name> $2"; exit 42;;
esac
;;
start|stop)
sudo_or_exit
case $2 in
''|--|containers) lxc_cmd "$1" ;;
${LXC_HOST_PREFIX}-*)
! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42
info_msg "lxc $1 $2"
lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] "
;;
*) usage "uknown or missing container <name> $2"; exit 42;;
esac
;;
show)
sudo_or_exit
case $2 in
suite)
case $3 in
${LXC_HOST_PREFIX}-*)
lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
| prefix_stdout "[${_BBlue}$3${_creset}] "
;;
*) show_suite;;
esac
;;
images) show_images ;;
config)
case $3 in
${LXC_HOST_PREFIX}-*)
! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
;;
*)
rst_title "container configurations"
echo
lxc list "$LXC_HOST_PREFIX-"
echo
lxc_cmd config show
;;
esac
;;
info)
case $3 in
${LXC_HOST_PREFIX}-*)
! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
;;
*)
rst_title "container info"
echo
lxc_cmd info
;;
esac
;;
*) usage "$_usage"; exit 42;;
esac
;;
__show)
# wrapped show commands, called once in each container
case $2 in
suite) lxc_suite_info ;;
esac
;;
cmd)
sudo_or_exit
shift
case $1 in
--) shift; lxc_exec "$@" ;;
${LXC_HOST_PREFIX}-*)
! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42
local name=$1
shift
lxc_exec_cmd "${name}" "$@"
;;
*) usage_containers "unknown container: $1" && exit 42
esac
;;
install)
sudo_or_exit
case $2 in
suite|base)
case $3 in
${LXC_HOST_PREFIX}-*)
! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2"
;;
''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;;
*) usage_containers "unknown container: $3" && exit 42
esac
;;
*) usage "$_usage"; exit 42 ;;
esac
;;
__install)
# wrapped install commands, called once in each container
# shellcheck disable=SC2119
case $2 in
suite) lxc_suite_install ;;
base) FORCE_TIMEOUT=0 lxc_install_base_packages ;;
esac
;;
doc)
echo
echo ".. generic utils/lxc.sh documentation"
;;
-*) usage "unknown option $1"; exit 42;;
*) usage "unknown or missing command $1"; exit 42;;
esac
}
build_all_containers() {
rst_title "Build all LXC containers of suite"
echo
usage_containers
lxc_copy_images_localy
lxc_init_all_containers
lxc_config_all_containers
lxc_boilerplate_all_containers
rst_title "install LXC base packages" section
echo
lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base
echo
lxc list "$LXC_HOST_PREFIX"
}
build_container() {
rst_title "Build container $1"
local remote_image
local container
local image
local boilerplate_script
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then
remote_image="${LXC_SUITE[i]}"
container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
image="${LXC_SUITE[i+1]}"
boilerplate_script="${image}_boilerplate"
boilerplate_script="${!boilerplate_script}"
break
fi
done
echo
if [ -z "$container" ]; then
err_msg "container $1 unknown"
usage_containers
return 42
fi
lxc_image_copy "${remote_image}" "${image}"
rst_title "init container" section
lxc_init_container "${image}" "${container}"
rst_title "configure container" section
lxc_config_container "${container}"
rst_title "run LXC boilerplate scripts" section
lxc_install_boilerplate "${container}" "$boilerplate_script"
echo
rst_title "install LXC base packages" section
lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \
| prefix_stdout "[${_BBlue}${container}${_creset}] "
echo
lxc list "$container"
}
remove_containers() {
rst_title "Remove all LXC containers of suite"
rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}"
echo
lxc list "$LXC_HOST_PREFIX-"
echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT
local default=Ny
[[ $FORCE_TIMEOUT = 0 ]] && default=Yn
if ask_yn "Do you really want to delete these containers" $default; then
for i in "${CONTAINERS[@]}"; do
lxc_delete_container "$i"
done
fi
echo
lxc list "$LXC_HOST_PREFIX-"
}
# images
# ------
lxc_copy_images_localy() {
rst_title "copy images" section
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}"
done
# lxc image list local: && wait_key
}
lxc_delete_images_localy() {
rst_title "Delete LXC images"
rst_para "local existing images"
echo
lxc image list local:
echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n"
if ask_yn "Do you really want to delete these images"; then
for i in "${LOCAL_IMAGES[@]}"; do
lxc_delete_local_image "$i"
done
fi
for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do
if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then
lxc_delete_local_image "$i"
fi
done
echo
lxc image list local:
}
show_images(){
rst_title "local images"
echo
lxc image list local:
echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n"
wait_key
for i in "${LOCAL_IMAGES[@]}"; do
if lxc_image_exists "$i"; then
info_msg "lxc image info ${_BBlue}${i}${_creset}"
lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
else
warn_msg "image ${_BBlue}$i${_creset} does not yet exists"
fi
done
}
# container
# ---------
show_suite(){
rst_title "LXC suite ($LXC_HOST_PREFIX-*)"
echo
lxc list "$LXC_HOST_PREFIX-"
echo
for i in "${CONTAINERS[@]}"; do
if ! lxc_exists "$i"; then
warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
else
lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
| prefix_stdout "[${_BBlue}${i}${_creset}] "
echo
fi
done
}
lxc_cmd() {
for i in "${CONTAINERS[@]}"; do
if ! lxc_exists "$i"; then
warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
else
info_msg "lxc $* $i"
lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
fi
done
}
lxc_exec_cmd() {
local name="$1"
shift
exit_val=
info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}"
lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*"
exit_val=$?
if [[ $exit_val -ne 0 ]]; then
warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}"
else
info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}"
fi
}
lxc_exec() {
for i in "${CONTAINERS[@]}"; do
if ! lxc_exists "$i"; then
warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
else
lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] "
fi
done
}
lxc_init_all_containers() {
rst_title "init all containers" section
local image_name
local container_name
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
done
}
lxc_config_all_containers() {
rst_title "configure all containers" section
for i in "${CONTAINERS[@]}"; do
lxc_config_container "${i}"
done
}
lxc_config_container() {
info_msg "[${_BBlue}$1${_creset}] configure container ..."
info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container"
# https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps
echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\
| lxc config set "$1" raw.idmap -
info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container"
# https://lxd.readthedocs.io/en/latest/instances/#type-disk
lxc config device add "$1" repo_share disk \
source="${REPO_ROOT}" \
path="${LXC_REPO_ROOT}" &>/dev/null
# lxc config show "$1" && wait_key
}
lxc_boilerplate_all_containers() {
rst_title "run LXC boilerplate scripts" section
local boilerplate_script
local image_name
for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
image_name="${LXC_SUITE[i+1]}"
boilerplate_script="${image_name}_boilerplate"
boilerplate_script="${!boilerplate_script}"
lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script"
if [[ -z "${boilerplate_script}" ]]; then
err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'"
fi
done
}
lxc_install_boilerplate() {
# usage: lxc_install_boilerplate <container-name> <string: shell commands ..>
#
# usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}"
local container_name="$1"
local boilerplate_script="$2"
info_msg "[${_BBlue}${container_name}${_creset}] init .."
if lxc start -q "${container_name}" &>/dev/null; then
sleep 5 # guest needs some time to come up and get an IP
fi
lxc_init_container_env "${container_name}"
info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .."
cat <<EOF | lxc exec "${container_name}" -- bash | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
rm -f "/.lxcenv.mk"
ln -s "${LXC_REPO_ROOT}/utils/makefile.lxc" "/.lxcenv.mk"
ls -l "/.lxcenv.mk"
EOF
info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .."
if lxc start -q "${container_name}" &>/dev/null; then
sleep 5 # guest needs some time to come up and get an IP
fi
if [[ -n "${boilerplate_script}" ]]; then
echo "${boilerplate_script}" \
| lxc exec "${container_name}" -- bash \
| prefix_stdout "[${_BBlue}${container_name}${_creset}] "
fi
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

16
utils/makefile.include
View File

@ -1,12 +1,25 @@
# -*- coding: utf-8; mode: makefile-gmake -*-
ifeq (,$(wildcard /.lxcenv.mk))
PHONY += lxc-activate lxc-purge
lxc-activate:
@$(MAKE) -s -f /share/searx/utils/makefile.lxc lxc-activate
lxc-purge:
$(Q)rm -rf ./lxc
else
include /.lxcenv.mk
endif
ifeq (,$(wildcard /.lxcenv.mk))
make-help:
else
make-help: lxc-help
endif
@echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
@echo ' make V=2 [targets] 2 => give reason for rebuild of target'
quiet_cmd_common_clean = CLEAN $@
cmd_common_clean = \
rm -rf tests/build ;\
find . -name '*.orig' -exec rm -f {} + ;\
find . -name '*.rej' -exec rm -f {} + ;\
find . -name '*~' -exec rm -f {} + ;\
@ -126,3 +139,4 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(ech
# printing commands
cmd = @$(echo-cmd) $(cmd_$(1))
.PHONY: $(PHONY)

29
utils/makefile.lxc Normal file
View File

@ -0,0 +1,29 @@
# -*- coding: utf-8; mode: makefile-gmake -*-
#
# LXC environment
# ===============
#
# To activate/deactivate LXC makefile environment in a container, set/unset link
# from root '/.lxcenv.mk' to *this* file::
#
# sudo make ./utils/makefile.lxc lxc-activate
# sudo make ./utils/makefile.lxc lxc-deactivate
LXC_ENV_FOLDER=lxc/$(shell hostname)/
lxc-help::
@echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)'
# If not activated, serve target 'lxc-activate' ..
ifeq (,$(wildcard /.lxcenv.mk))
PHONY += lxc-activate
lxc-activate:
ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk"
else
# .. and if activated, serve target 'lxc-deactivate'.
PHONY += lxc-deactivate
lxc-deactivate:
rm /.lxcenv.mk
endif
.PHONY: $(PHONY)

8
utils/makefile.python
View File

@ -8,9 +8,9 @@ export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
export PY_ENV PYDIST PYBUILD
# folder where the python distribution takes place
PYDIST ?= ./py_dist
PYDIST = ./$(LXC_ENV_FOLDER)dist
# folder where the python intermediate build files take place
PYBUILD ?= ./py_build
PYBUILD = ./$(LXC_ENV_FOLDER)build
# python version to use
PY ?=3
# $(PYTHON) points to the python interpreter from the OS! The python from the
@ -30,8 +30,7 @@ PYLINT_RC ?= .pylintrc
TEST_FOLDER ?= ./tests
TEST ?= .
VTENV_OPTS = "--no-site-packages"
PY_ENV = ./local/py$(PY)
PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY)
PY_ENV_BIN = $(PY_ENV)/bin
PY_ENV_ACT = . $(PY_ENV_BIN)/activate
@ -41,6 +40,7 @@ ifeq ($(OS),Windows_NT)
PY_ENV_ACT = $(PY_ENV_BIN)/activate
endif
VTENV_OPTS ?=
ifeq ($(PYTHON),python)
VIRTUALENV = virtualenv
else

26
utils/makefile.sphinx
View File

@ -1,17 +1,19 @@
# -*- coding: utf-8; mode: makefile-gmake -*-
export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST
# You can set these variables from the command line.
SPHINXOPTS ?=
SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
SPHINX_CONF ?= conf.py
DOCS_FOLDER ?= docs
DOCS_BUILD ?= build/docs
DOCS_DIST ?= dist/docs
DOCS_FOLDER = ./docs
DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs
DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs
GH_PAGES ?= gh-pages
BOOKS_FOLDER ?= docs
BOOKS_DIST ?= dist/books
BOOKS_FOLDER = ./docs
BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books
ifeq ($(KBUILD_VERBOSE),1)
SPHINX_VERBOSE = "-v"
@ -54,11 +56,13 @@ docs-help:
# requirements
# ------------------------------------------------------------------------------
sphinx-doc: $(PY_ENV)
sphinx-doc-prebuilds:: $(PY_ENV)
sphinx-doc: sphinx-doc-prebuilds
@echo "PYENV installing Sphinx$(SPHINXVERS)"
$(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)'
sphinx-live: $(PY_ENV)
sphinx-live: sphinx-doc-prebuilds
@echo "PYENV installing Sphinx$(SPHINXVERS)"
$(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild
@ -113,7 +117,7 @@ quiet_cmd_sphinx_clean = CLEAN $@
# targets
# ------------------------------------------------------------------------------
# build PDF of whole documentation in: $(DOCS_DIST)/pdf
# build PDF of whole documentation in: $(DOCS_DIST)/pdf
PHONY += sphinx-pdf
sphinx-pdf: sphinx-latex
@ -154,7 +158,7 @@ $(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST)
-b html \
-c $(DOCS_FOLDER) \
-d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
$(patsubst books/%.html,%,$@) \
$(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \
$(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
@echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
@ -166,7 +170,7 @@ $(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST)
-b html \
-c $(DOCS_FOLDER) \
-d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
$(patsubst books/%.live,%,$@) \
$(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \
$(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
$(BOOKS_PDF): %.pdf : %.latex
@ -182,7 +186,7 @@ $(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST)
-b latex \
-c $(DOCS_FOLDER) \
-d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
$(patsubst books/%.latex,%,$@) \
$(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \
$(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
@echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"

546
utils/morty.sh Executable file
View File

@ -0,0 +1,546 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
source_dot_config
SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
source "${REPO_ROOT}/utils/lxc-searx.env"
in_container && lxc_set_suite_env
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"
PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}"
# shellcheck disable=SC2034
MORTY_TIMEOUT=5
SERVICE_NAME="morty"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_ENV_DEBUG=false
GO_ENV="${SERVICE_HOME}/.go_env"
GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
GO_TAR=$(basename "$GO_PKG_URL")
# shellcheck disable=SC2034
CONFIG_FILES=()
# Apache Settings
APACHE_MORTY_SITE="morty.conf"
NGINX_MORTY_SITE="morty.conf"
# ----------------------------------------------------------------------------
usage() {
# ----------------------------------------------------------------------------
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|user]
$(basename "$0") update [morty]
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off|new-key]
$(basename "$0") apache [install|remove]
$(basename "$0") nginx [install|remove]
$(basename "$0") info [searx]
shell
start interactive shell from user ${SERVICE_USER}
install / remove
all: complete setup of morty service
user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
update morty
Update morty installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
show service status and log
option
set one of the available options
:new-key: set new morty key
apache : ${PUBLIC_URL_MORTY}
:install: apache site with a reverse proxy (ProxyPass)
:remove: apache site ${APACHE_MORTY_SITE}
nginx (${PUBLIC_URL_MORTY})
:install: nginx site with a reverse proxy (ProxyPass)
:remove: nginx site ${NGINX_MORTY_SITE}
If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY}
MORTY_LISTEN: ${MORTY_LISTEN}
SERVICE_USER: ${SERVICE_USER}
EOF
if in_container; then
# in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container URL (IPv6): http://[${ip#*|}]:3000/"
else
# IPv4:
echo " container URL (IPv4): http://${ip#*|}:3000/"
fi
done
fi
echo
info_searx
[[ -n ${1} ]] && err_msg "$1"
}
info_searx() {
# shellcheck disable=SC1117
cat <<EOF
To activate result and image proxy in searx, edit settings.yml (read:
${DOCS_URL}/admin/morty.html)::
result_proxy:
url : ${PUBLIC_URL_MORTY}
server:
image_proxy : True
EOF
}
main() {
required_commands \
sudo install git wget curl \
|| exit
local _usage="ERROR: unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SERVICE_NAME" part
sudo_or_exit
case $2 in
all) install_all ;;
user) assert_user ;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
morty) update_morty ;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service) systemd_activate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) systemd_deactivate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
install) install_nginx_site ;;
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
info)
case $2 in
searx) info_searx ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
new-key) set_new_key ;;
debug-on) enable_debug ;;
debug-off) disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;;
esac
}
install_all() {
MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
rst_title "Install $SERVICE_NAME (service)"
assert_user
wait_key
install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"
wait_key
install_morty
wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
wait_key
if ! service_is_available "http://${MORTY_LISTEN}" ; then
err_msg "Morty does not listening on: http://${MORTY_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
elif nginx_is_installed; then
info_msg "nginx is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_nginx_site
fi
fi
info_searx
if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then
"${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
"${REPO_ROOT}/utils/searx.sh" option image-proxy-on
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
rst_para "\
It goes without saying that this script can only be used to remove
installations that were installed with this script."
if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
drop_service_account "${SERVICE_USER}"
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Web content sanitizer proxy' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
export SERVICE_HOME
echo "export SERVICE_HOME=$SERVICE_HOME"
cat > "$GO_ENV" <<EOF
export GOPATH=\$HOME/go-apps
export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin
EOF
echo "Environment $GO_ENV has been setup."
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile
EOF
}
morty_is_installed() {
[[ -f $SERVICE_HOME/go-apps/bin/morty ]]
}
_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "
install_morty() {
rst_title "Install morty in user's ~/go-apps" section
echo
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
go get -v -u github.com/asciimoo/morty
EOF
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
cd \$GOPATH/src/github.com/asciimoo/morty
go test
go test -benchmem -bench .
EOF
}
update_morty() {
rst_title "Update morty" section
echo
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
go get -v -u github.com/asciimoo/morty
EOF
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"
cd \$GOPATH/src/github.com/asciimoo/morty
go test
go test -benchmem -bench .
EOF
}
set_service_env_debug() {
# usage: set_service_env_debug [false|true]
# shellcheck disable=SC2034
local SERVICE_ENV_DEBUG="${1:-false}"
if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
fi
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
MORTY_LISTEN : ${MORTY_LISTEN}
EOF
if service_account_is_available "$SERVICE_USER"; then
info_msg "service account $SERVICE_USER available."
else
err_msg "service account $SERVICE_USER not available!"
fi
if go_is_available "$SERVICE_USER"; then
info_msg "~$SERVICE_USER: go is installed"
else
err_msg "~$SERVICE_USER: go is not installed"
fi
if morty_is_installed; then
info_msg "~$SERVICE_USER: morty app is installed"
else
err_msg "~$SERVICE_USER: morty app is not installed!"
fi
if ! service_is_available "http://${MORTY_LISTEN}" ; then
err_msg "Morty does not listening on: http://${MORTY_LISTEN}"
echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .."
wait_key
fi
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL_MORTY}"
info_msg "morty URL --> http://${MORTY_LISTEN}"
fi
local _debug_on
if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then
enable_debug
_debug_on=1
else
systemctl --no-pager -l status "${SERVICE_NAME}"
fi
echo
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
journalctl -f -u "${SERVICE_NAME}"
done
if [[ $_debug_on == 1 ]]; then
FORCE_SELECTION=Y disable_debug
fi
return 0
}
enable_debug() {
warn_msg "Do not enable debug in production enviroments!!"
info_msg "Enabling debug option needs to reinstall systemd service!"
set_service_env_debug true
}
disable_debug() {
info_msg "Disabling debug option needs to reinstall systemd service!"
set_service_env_debug false
}
set_new_key() {
rst_title "Set morty key"
echo
MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
info_msg "morty key: '${MORTY_KEY}'"
warn_msg "this will need to reinstall services .."
MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
"${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
"${REPO_ROOT}/utils/searx.sh" option image-proxy-on
}
install_apache_site() {
rst_title "Install Apache site $APACHE_MORTY_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})"
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
apache_install_site "${APACHE_MORTY_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site $APACHE_MORTY_SITE"
rst_para "\
This removes apache site ${APACHE_MORTY_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
apache_remove_site "$APACHE_MORTY_SITE"
}
install_nginx_site() {
rst_title "Install nginx site $NGINX_MORTY_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"
! nginx_is_installed && err_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_nginx
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
# shellcheck disable=SC2034
SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
# shellcheck disable=SC2034
SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
nginx_install_app "${NGINX_MORTY_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
fi
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_MORTY_SITE"
rst_para "\
This removes nginx site ${NGINX_MORTY_SITE}."
! nginx_is_installed && err_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
nginx_remove_site "$NGINX_MORTY_SITE"
}
rst-doc() {
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\""
echo -e "\n.. START install systemd unit"
cat <<EOF
.. tabs::
.. group-tab:: systemd
.. code:: bash
EOF
eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
echo -e "\n.. END install systemd unit"
# for DIST_NAME in ubuntu-20.04 arch fedora; do
# (
# DIST_ID=${DIST_NAME%-*}
# DIST_VERS=${DIST_NAME#*-}
# [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
# # ...
# )
# done
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

869
utils/searx.sh Executable file
View File

@ -0,0 +1,869 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2001
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
source_dot_config
source "${REPO_ROOT}/utils/lxc-searx.env"
in_container && lxc_set_suite_env
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}"
SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \
| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/
SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \
| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }"
SERVICE_NAME="searx"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
GIT_BRANCH="${GIT_BRANCH:-master}"
SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"
SEARX_SRC="${SERVICE_HOME}/searx-src"
SEARX_SETTINGS_PATH="/etc/searx/settings.yml"
SEARX_UWSGI_APP="searx.ini"
# shellcheck disable=SC2034
SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket"
# apt packages
SEARX_PACKAGES_debian="\
virtualenv python3-dev python3-babel python3-venv
uwsgi uwsgi-plugin-python3
git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
shellcheck"
BUILD_PACKAGES_debian="\
firefox graphviz imagemagick texlive-xetex librsvg2-bin
texlive-latex-recommended texlive-extra-utils ttf-dejavu
latexmk"
# pacman packages
SEARX_PACKAGES_arch="\
python-virtualenv python python-pip python-lxml python-babel
uwsgi uwsgi-plugin-python
git base-devel libxml2
shellcheck"
BUILD_PACKAGES_arch="\
firefox graphviz imagemagick texlive-bin extra/librsvg
texlive-core texlive-latexextra ttf-dejavu"
# dnf packages
SEARX_PACKAGES_fedora="\
virtualenv python python-pip python-lxml python-babel
uwsgi uwsgi-plugin-python3
git @development-tools libxml2
ShellCheck"
BUILD_PACKAGES_fedora="\
firefox graphviz graphviz-gd ImageMagick librsvg2-tools
texlive-xetex-bin texlive-collection-fontsrecommended
texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts
dejavu-sans-mono-fonts"
case $DIST_ID-$DIST_VERS in
ubuntu-16.04|ubuntu-18.04)
SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi"
;;
ubuntu-20.04)
# https://askubuntu.com/a/1224710
SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3"
BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
;;
ubuntu-*|debian-*)
SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
;;
arch-*)
SEARX_PACKAGES="${SEARX_PACKAGES_arch}"
BUILD_PACKAGES="${BUILD_PACKAGES_arch}"
;;
fedora-*)
SEARX_PACKAGES="${SEARX_PACKAGES_fedora}"
BUILD_PACKAGES="${BUILD_PACKAGES_fedora}"
;;
esac
# Apache Settings
APACHE_SEARX_SITE="searx.conf"
# shellcheck disable=SC2034
CONFIG_FILES=(
"${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}"
)
# shellcheck disable=SC2034
CONFIG_BACKUP_ENCRYPTED=(
"${SEARX_SETTINGS_PATH}"
)
# ----------------------------------------------------------------------------
usage() {
# ----------------------------------------------------------------------------
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|user|searx-src|pyenv|uwsgi|packages|buildhost]
$(basename "$0") update [searx]
$(basename "$0") remove [all|user|pyenv|searx-src]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>]
$(basename "$0") apache [install|remove]
shell
start interactive shell from user ${SERVICE_USER}
install / remove
:all: complete (de-) installation of searx service
:user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
:searx-src: clone $GIT_URL
:pyenv: create/remove virtualenv (python) in $SEARX_PYENV
:uwsgi: install searx uWSGI application
:settings: reinstall settings from ${REPO_ROOT}/searx/settings.yml
:packages: install needed packages from OS package manager
:buildhost: install packages from OS package manager needed by buildhosts
update searx
Update searx installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
run some small tests and inspect service's status and log
option
set one of the available options
apache
:install: apache site with the searx uwsgi app
:remove: apache site ${APACHE_FILTRON_SITE}
searx settings: ${SEARX_SETTINGS_PATH}
If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::
PUBLIC_URL : ${PUBLIC_URL}
SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
SERVICE_USER : ${SERVICE_USER}
SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP}
EOF
if in_container; then
# searx is listening on 127.0.0.1 and not available from outside container
# in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \
"${_BBlack}inside${_creset} the container."
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container (IPv6): [${ip#*|}]"
else
# IPv4:
echo " container (IPv4): ${ip#*|}"
fi
done
fi
[[ -n ${1} ]] && err_msg "$1"
}
main() {
required_commands \
sudo systemctl install git wget curl \
|| exit
local _usage="unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SEARX_INSTANCE_NAME" part
sudo_or_exit
case $2 in
all) install_all ;;
user) assert_user ;;
pyenv) create_pyenv ;;
searx-src) clone_searx ;;
settings) install_settings ;;
uwsgi)
install_searx_uwsgi
if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
fi
;;
packages)
pkg_install "$SEARX_PACKAGES"
;;
buildhost)
pkg_install "$SEARX_PACKAGES"
pkg_install "$BUILD_PACKAGES"
;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
searx) update_searx;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}";;
pyenv) remove_pyenv ;;
searx-src) remove_searx ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service)
activate_service ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) deactivate_service ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
debug-on) echo; enable_debug ;;
debug-off) echo; disable_debug ;;
result-proxy) set_result_proxy "$3" "$4" ;;
image-proxy-on) enable_image_proxy ;;
image-proxy-off) disable_image_proxy ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc;;
*) usage "unknown or missing command $1"; exit 42;;
esac
}
_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} "
install_all() {
rst_title "Install $SEARX_INSTANCE_NAME (service)"
pkg_install "$SEARX_PACKAGES"
wait_key
assert_user
wait_key
clone_searx
wait_key
create_pyenv
wait_key
install_settings
wait_key
test_local_searx
wait_key
install_searx_uwsgi
if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
update_searx() {
rst_title "Update searx instance"
echo
tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC}
git checkout -B "$GIT_BRANCH"
git pull
pip install -U pip
pip install -U setuptools
pip install -U wheel
pip install -U -e .
EOF
install_settings
uWSGI_restart "$SEARX_UWSGI_APP"
}
remove_all() {
rst_title "De-Install $SEARX_INSTANCE_NAME (service)"
rst_para "\
It goes without saying that this script can only be used to remove
installations that were installed with this script."
if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then
return
fi
remove_searx_uwsgi
drop_service_account "${SERVICE_USER}"
remove_settings
wait_key
if service_is_available "${PUBLIC_URL}"; then
MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Privacy-respecting metasearch engine' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
#SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
#export SERVICE_HOME
#echo "export SERVICE_HOME=$SERVICE_HOME"
}
clone_is_available() {
[[ -f "$SEARX_SRC/.git/config" ]]
}
# shellcheck disable=SC2164
clone_searx() {
rst_title "Clone searx sources" section
echo
if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then
die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT"
fi
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"
if [[ ! "${SERVICE_HOME}" ]]; then
err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"
return 42
fi
export SERVICE_HOME
git_clone "$REPO_ROOT" "$SEARX_SRC" \
"$GIT_BRANCH" "$SERVICE_USER"
pushd "${SEARX_SRC}" > /dev/null
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
cd "${SEARX_SRC}"
git remote set-url origin ${GIT_URL}
git config user.email "$ADMIN_EMAIL"
git config user.name "$ADMIN_NAME"
git config --list
EOF
popd > /dev/null
}
install_settings() {
rst_title "${SEARX_SETTINGS_PATH}" section
if ! clone_is_available; then
err_msg "you have to install searx first"
exit 42
fi
mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})"
if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then
info_msg "install settings ${REPO_ROOT}/searx/settings.yml"
info_msg " --> ${SEARX_SETTINGS_PATH}"
cp "${REPO_ROOT}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
configure_searx
return
fi
rst_para "Diff between origin's setting file (+) and current (-):"
echo
$DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SRC}/searx/settings.yml"
local action
choose_one action "What should happen to the settings file? " \
"keep configuration unchanged" \
"use origin settings" \
"start interactiv shell"
case $action in
"keep configuration unchanged")
info_msg "leave settings file unchanged"
;;
"use origin settings")
backup_file "${SEARX_SETTINGS_PATH}"
info_msg "install origin settings"
cp "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
;;
"start interactiv shell")
backup_file "${SEARX_SETTINGS_PATH}"
echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
sudo -H -i
rst_para 'Diff between new setting file (-) and current (+):'
echo
$DIFF_CMD "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}"
wait_key
;;
esac
}
remove_settings() {
rst_title "remove searx settings" section
echo
info_msg "delete ${SEARX_SETTINGS_PATH}"
rm -f "${SEARX_SETTINGS_PATH}"
}
remove_searx() {
rst_title "Drop searx sources" section
if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then
rm -rf "$SEARX_SRC"
else
rst_para "Leave searx sources unchanged."
fi
}
pyenv_is_available() {
[[ -f "${SEARX_PYENV}/bin/activate" ]]
}
create_pyenv() {
rst_title "Create virtualenv (python)" section
echo
if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then
err_msg "to create pyenv for searx, searx has to be cloned first"
return 42
fi
info_msg "create pyenv in ${SEARX_PYENV}"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
rm -rf "${SEARX_PYENV}"
python3 -m venv "${SEARX_PYENV}"
grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \
|| echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile
EOF
info_msg "inspect python's virtual environment"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
command -v python && python --version
EOF
wait_key
info_msg "install needed python packages"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
pip install -U pip
pip install -U setuptools
pip install -U wheel
pip install -U -e .
cd ${SEARX_SRC}
pip install -e .
EOF
}
remove_pyenv() {
rst_title "Remove virtualenv (python)" section
if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then
return
fi
info_msg "remove pyenv activation from ~/.profile"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##
mv ~/.profile.## ~/.profile
EOF
rm -rf "${SEARX_PYENV}"
}
configure_searx() {
rst_title "Configure searx" section
rst_para "Setup searx config located at $SEARX_SETTINGS_PATH"
echo
tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC}
sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH"
sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH"
EOF
}
test_local_searx() {
rst_title "Testing searx instance localy" section
echo
if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then
err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you"
err_msg "should stop that service before starting local tests!"
if ! ask_yn "Continue with local tests?"; then
return
fi
fi
sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}"
cd ${SEARX_SRC}
timeout 10 python searx/webapp.py &
sleep 3
curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP
EOF
sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
}
install_searx_uwsgi() {
rst_title "Install searx's uWSGI app (searx.ini)" section
echo
install_uwsgi
uWSGI_install_app "$SEARX_UWSGI_APP"
}
remove_searx_uwsgi() {
rst_title "Remove searx's uWSGI app (searx.ini)" section
echo
uWSGI_remove_app "$SEARX_UWSGI_APP"
}
activate_service() {
rst_title "Activate $SEARX_INSTANCE_NAME (service)" section
echo
uWSGI_enable_app "$SEARX_UWSGI_APP"
uWSGI_restart "$SEARX_UWSGI_APP"
}
deactivate_service() {
rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section
echo
uWSGI_disable_app "$SEARX_UWSGI_APP"
uWSGI_restart "$SEARX_UWSGI_APP"
}
enable_image_proxy() {
info_msg "try to enable image_proxy ..."
tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC}
sed -i -e "s/image_proxy : False/image_proxy : True/g" "$SEARX_SETTINGS_PATH"
EOF
uWSGI_restart "$SEARX_UWSGI_APP"
}
disable_image_proxy() {
info_msg "try to enable image_proxy ..."
tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC}
sed -i -e "s/image_proxy : True/image_proxy : False/g" "$SEARX_SETTINGS_PATH"
EOF
uWSGI_restart "$SEARX_UWSGI_APP"
}
enable_debug() {
warn_msg "Do not enable debug in production enviroments!!"
info_msg "try to enable debug mode ..."
tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC}
sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH"
EOF
uWSGI_restart "$SEARX_UWSGI_APP"
}
disable_debug() {
info_msg "try to disable debug mode ..."
tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC}
sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH"
EOF
uWSGI_restart "$SEARX_UWSGI_APP"
}
set_result_proxy() {
# usage: set_result_proxy <URL> [<key>]
info_msg "try to set result proxy: '$1' ($2)"
cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak"
_set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}"
}
_set_result_proxy() {
local line
local stage=0
local url=" url: $1"
local key=" key: !!binary \"$2\""
if [[ -z $2 ]]; then
key=
fi
while IFS= read -r line
do
if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then
if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then
if [[ $stage = 0 ]]; then
stage=1
echo "result_proxy:"
continue
elif [[ $stage = 2 ]]; then
continue
fi
fi
fi
if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then
if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then
[[ $stage = 1 ]] && echo "$url"
continue
elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then
[[ $stage = 1 ]] && [[ -n $key ]] && echo "$key"
continue
elif [[ $line =~ ^[[:space:]]*$ ]]; then
stage=2
fi
fi
echo "$line"
done < "${SEARX_SETTINGS_PATH}.bak"
}
function has_substring() {
[[ "$1" != "${2/$1/}" ]]
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
PUBLIC_URL : ${PUBLIC_URL}
SEARX_URL_PATH : ${SEARX_URL_PATH}
SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
SEARX_INTERNAL_HTTP : ${SEARX_INTERNAL_HTTP}
EOF
if service_account_is_available "$SERVICE_USER"; then
info_msg "Service account $SERVICE_USER exists."
else
err_msg "Service account $SERVICE_USER does not exists!"
fi
if pyenv_is_available; then
info_msg "~$SERVICE_USER: python environment is available."
else
err_msg "~$SERVICE_USER: python environment is not available!"
fi
if clone_is_available; then
info_msg "~$SERVICE_USER: Searx software is installed."
else
err_msg "~$SERVICE_USER: Missing searx software!"
fi
if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then
info_msg "uWSGI app $SEARX_UWSGI_APP is enabled."
else
err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!"
fi
uWSGI_app_available "$SEARX_UWSGI_APP" \
|| err_msg "uWSGI app $SEARX_UWSGI_APP not available!"
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL}"
info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}"
fi
if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!"
MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\
wait_key
fi
if ! service_is_available "${PUBLIC_URL}"; then
warn_msg "Public service at ${PUBLIC_URL} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
local _debug_on
if ask_yn "Enable searx debug mode?"; then
enable_debug
_debug_on=1
fi
echo
case $DIST_ID-$DIST_VERS in
ubuntu-*|debian-*)
systemctl --no-pager -l status "${SERVICE_NAME}"
;;
arch-*)
systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}"
;;
fedora-*)
systemctl --no-pager -l status uwsgi
;;
esac
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
case $DIST_ID-$DIST_VERS in
ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;;
arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;;
fedora-*) journalctl -f -u uwsgi ;;
esac
done
if [[ $_debug_on == 1 ]]; then
disable_debug
fi
return 0
}
install_apache_site() {
rst_title "Install Apache site $APACHE_SEARX_SITE"
rst_para "\
This installs the searx uwsgi app as apache site. If your server is public to
the internet, you should instead use a reverse proxy (filtron) to block
excessively bot queries."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
rst_title "Install searx's uWSGI app (searx.ini)" section
echo
uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site ${APACHE_SEARX_SITE}"
rst_para "\
This removes apache site ${APACHE_SEARX_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
apache_remove_site "${APACHE_SEARX_SITE}"
rst_title "Remove searx's uWSGI app (searx.ini)" section
echo
uWSGI_remove_app "$SEARX_UWSGI_APP"
}
rst-doc() {
local debian="${SEARX_PACKAGES_debian}"
local arch="${SEARX_PACKAGES_arch}"
local fedora="${SEARX_PACKAGES_fedora}"
local debian_build="${BUILD_PACKAGES_debian}"
local arch_build="${BUILD_PACKAGES_arch}"
local fedora_build="${BUILD_PACKAGES_fedora}"
debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')"
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\""
# I use ubuntu-20.04 here to demonstrate that versions are also suported,
# normaly debian-* and ubuntu-* are most the same.
for DIST_NAME in ubuntu-20.04 arch fedora; do
(
DIST_ID=${DIST_NAME%-*}
DIST_VERS=${DIST_NAME#*-}
[[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
uWSGI_distro_setup
echo -e "\n.. START searx uwsgi-description $DIST_NAME"
case $DIST_ID-$DIST_VERS in
ubuntu-*|debian-*) cat <<EOF
# init.d --> /usr/share/doc/uwsgi/README.Debian.gz
# For uWSGI debian uses the LSB init process, this might be changed
# one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}
enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/
start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*}
restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*}
stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*}
disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
EOF
;;
arch-*) cat <<EOF
# systemd --> /usr/lib/systemd/system/uwsgi@.service
# For uWSGI archlinux uses systemd template units, see
# - http://0pointer.de/blog/projects/instances.html
# - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*}
start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*}
restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*}
stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*}
disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*}
EOF
;;
fedora-*) cat <<EOF
# systemd --> /usr/lib/systemd/system/uwsgi.service
# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see
# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP}
EOF
;;
esac
echo -e ".. END searx uwsgi-description $DIST_NAME"
echo -e "\n.. START searx uwsgi-appini $DIST_NAME"
eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\""
echo -e "\n.. END searx uwsgi-appini $DIST_NAME"
)
done
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

48
utils/site-python/sphinx_build_tools.py Normal file
View File

@ -0,0 +1,48 @@
# -*- coding: utf-8; mode: python -*-
"""Implement some sphinx-build tools.
"""
import os
import sys
from sphinx.util.pycompat import execfile_
# ------------------------------------------------------------------------------
def load_sphinx_config(namespace):
# ------------------------------------------------------------------------------
u"""Load an additional configuration file into *namespace*.
The name of the configuration file is taken from the environment
``SPHINX_CONF``. The external configuration file extends (or overwrites) the
configuration values from the origin ``conf.py``. With this you are able to
maintain *build themes*. To your docs/conf.py add::
from sphinx_build_tools import load_sphinx_config
...
# Since loadConfig overwrites settings from the global namespace, it has to be
# the last statement in the conf.py file
load_sphinx_config(globals())
"""
config_file = os.environ.get("SPHINX_CONF", None)
if (config_file is not None
and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ):
config_file = os.path.abspath(config_file)
if os.path.isfile(config_file):
sys.stdout.write(
"load additional sphinx-config: %s\n"
% config_file)
config = namespace.copy()
config['__file__'] = config_file
execfile_(config_file, config)
del config['__file__']
namespace.update(config)
else:
sys.stderr.write(
"WARNING: additional sphinx-config not found: %s\n"
% config_file)

1
utils/templates/etc/apache2 Symbolic link
View File

@ -0,0 +1 @@
httpd

129
utils/templates/etc/filtron/rules.json Normal file
View File

@ -0,0 +1,129 @@
[
{
"name": "roboagent limit",
"filters": [
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)"
],
"limit": 0,
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "botlimit",
"filters": [
"Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
],
"limit": 0,
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "suspiciously frequent IP",
"filters": [],
"interval": 600,
"limit": 30,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions":[
{"name":"log"}
]
},
{
"name": "search request",
"filters": [
"Param:q",
"Path=^(/|/search)$"
],
"interval": 61,
"limit": 999,
"subrules": [
{
"name": "missing Accept-Language",
"filters": ["!Header:Accept-Language"],
"limit": 0,
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "suspiciously Connection=close header",
"filters": ["Header:Connection=close"],
"limit": 0,
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "IP limit",
"interval": 61,
"limit": 9,
"stop": true,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "rss/json limit",
"filters": [
"Param:format=(csv|json|rss)"
],
"interval": 121,
"limit": 2,
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "useragent limit",
"interval": 61,
"limit": 199,
"aggregations": [
"Header:User-Agent"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
}
]
}
]

28
utils/templates/etc/httpd/sites-available/morty.conf Normal file
View File

@ -0,0 +1,28 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog
# CustomLog /dev/null combined env=dontlog
<Location ${PUBLIC_URL_PATH_MORTY} >
<IfModule mod_security2.c>
SecRuleEngine Off
</IfModule>
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass http://${MORTY_LISTEN}
RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY}
</Location>

33
utils/templates/etc/httpd/sites-available/searx.conf:filtron Normal file
View File

@ -0,0 +1,33 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog
# CustomLog /dev/null combined env=dontlog
# SecRuleRemoveById 981054
# SecRuleRemoveById 981059
# SecRuleRemoveById 981060
# SecRuleRemoveById 950907
<Location ${FILTRON_URL_PATH} >
<IfModule mod_security2.c>
SecRuleEngine Off
</IfModule>
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass http://${FILTRON_LISTEN}
RequestHeader set X-Script-Name ${FILTRON_URL_PATH}
</Location>

27
utils/templates/etc/httpd/sites-available/searx.conf:uwsgi Normal file
View File

@ -0,0 +1,27 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so
# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog
# CustomLog /dev/null combined env=dontlog
<Location ${SEARX_URL_PATH}>
<IfModule mod_security2.c>
SecRuleEngine Off
</IfModule>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/
</Location>

11
utils/templates/etc/nginx/default.apps-available/morty.conf Normal file
View File

@ -0,0 +1,11 @@
# https://example.org/morty
location /morty {
proxy_pass http://127.0.0.1:3000/;
proxy_set_header Host \$http_host;
proxy_set_header Connection \$http_connection;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Scheme \$scheme;
}

16
utils/templates/etc/nginx/default.apps-available/searx.conf:filtron Normal file
View File

@ -0,0 +1,16 @@
# https://example.org/searx
location ${SEARX_URL_PATH} {
proxy_pass http://127.0.0.1:4004/;
proxy_set_header Host \$http_host;
proxy_set_header Connection \$http_connection;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Scheme \$scheme;
proxy_set_header X-Script-Name ${SEARX_URL_PATH};
}
location ${SEARX_URL_PATH}/static {
alias ${SEARX_SRC}/searx/static;
}

80
utils/templates/etc/uwsgi/apps-archlinux/searx.ini Normal file
View File

@ -0,0 +1,80 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
logger = systemd
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
# socket = /run/uwsgi/app/searx/socket

80
utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket Normal file
View File

@ -0,0 +1,80 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
logger = systemd
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
socket = /run/uwsgi/app/searx/socket

79
utils/templates/etc/uwsgi/apps-available/searx.ini Normal file
View File

@ -0,0 +1,79 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python3,http
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
# socket = /run/uwsgi/app/searx/socket

79
utils/templates/etc/uwsgi/apps-available/searx.ini:socket Normal file
View File

@ -0,0 +1,79 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
# searx configuration (settings.yml)
env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
# disable logging for privacy
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python3,http
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
#
# On some distributions you need to create the app folder for the sockets::
#
# mkdir -p /run/uwsgi/app/searx
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx
#
socket = /run/uwsgi/app/searx/socket

29
utils/templates/lib/systemd/system/filtron.service Normal file
View File

@ -0,0 +1,29 @@
[Unit]
Description=${SERVICE_NAME}
After=syslog.target
After=network.target
[Service]
Type=simple
User=${SERVICE_USER}
Group=${SERVICE_GROUP}
WorkingDirectory=${SERVICE_HOME}
ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'
Restart=always
Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}
# Some distributions may not support these hardening directives. If you cannot
# start the service due to an unknown option, comment out the ones not supported
# by your version of systemd.
ProtectSystem=full
PrivateDevices=yes
PrivateTmp=yes
NoNewPrivileges=true
[Install]
WantedBy=multi-user.target

29
utils/templates/lib/systemd/system/morty.service Normal file
View File

@ -0,0 +1,29 @@
[Unit]
Description=${SERVICE_NAME}
After=syslog.target
After=network.target
[Service]
Type=simple
User=${SERVICE_USER}
Group=${SERVICE_GROUP}
WorkingDirectory=${SERVICE_HOME}
ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '${MORTY_KEY}' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT}
Restart=always
Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG}
# Some distributions may not support these hardening directives. If you cannot
# start the service due to an unknown option, comment out the ones not supported
# by your version of systemd.
ProtectSystem=full
PrivateDevices=yes
PrivateTmp=yes
NoNewPrivileges=true
[Install]
WantedBy=multi-user.target