diff --git a/.config.sh b/.config.sh new file mode 100644 index 00000000..4eff5f4c --- /dev/null +++ b/.config.sh @@ -0,0 +1,55 @@ +# -*- coding: utf-8; mode: sh -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck shell=bash disable=SC2034 +# +# This environment is used by ./utils scripts like filtron.sh or searx.sh. The +# default values are *most flexible* and *best maintained*, you normally not +# need to change the defaults (except PUBLIC_URL). +# +# Before you change any value here you have to uninstall any previous +# installation. Further is it recommended to backup your changes simply by +# adding them to you local brand (git branch):: +# +# git add .config + +# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx" +# The default is taken from ./utils/brand.env. + +PUBLIC_URL="${SEARX_URL}" + +if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then + # hint: Linux containers do not have DNS entries, lets use IPs + PUBLIC_URL="http://$(primary_ip)/searx" +fi + +# searx.sh +# --------- + +# SEARX_INTERNAL_URL="127.0.0.1:8888" + +# Only change, if you maintain a searx brand in your searx fork. +# GIT_BRANCH="${GIT_BRANCH:-master}" + +# filtron.sh +# ---------- + +# FILTRON_API="127.0.0.1:4005" +# FILTRON_LISTEN="127.0.0.1:4004" +# FILTRON_TARGET="127.0.0.1:8888" + +# morty.sh +# -------- + +# morty listen address +# MORTY_LISTEN="127.0.0.1:3000" +# PUBLIC_URL_PATH_MORTY="/morty/" + +# system services +# --------------- + +# Common $HOME folder of the service accounts +# SERVICE_HOME_BASE="/usr/local" + +# **experimental**: Set SERVICE_USER to run all services by one account, but be +# aware that removing discrete components might conflict! +# SERVICE_USER=searx diff --git a/.gitignore b/.gitignore index 1c82be41..e56a575a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # to sync with .dockerignore .coverage coverage/ +cache/ .installed.cfg engines.cfg env diff --git a/Makefile b/Makefile index abd0eddb..d10bd9db 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,31 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +.DEFAULT_GOAL=help +# START Makefile setup export GIT_URL=https://github.com/asciimoo/searx +export GIT_BRANCH=master export SEARX_URL=https://searx.me export DOCS_URL=https://asciimoo.github.io/searx +# END Makefile setup + +include utils/makefile.include PYOBJECTS = searx DOC = docs PY_SETUP_EXTRAS ?= \[test\] -PYDIST=./dist/py -PYBUILD=./build/py - -include utils/makefile.include include utils/makefile.python include utils/makefile.sphinx all: clean install -PHONY += help -help: +PHONY += help-min help-all help + +help: help-min + @echo '' + @echo 'to get more help: make help-all' + +help-min: @echo ' test - run developer tests' @echo ' docs - build documentation' @echo ' docs-live - autobuild HTML documentation while editing' @@ -33,9 +40,18 @@ help: @echo ' docker - build Docker image' @echo ' node.env - download & install npm dependencies locally' @echo '' - @$(MAKE) -s -f utils/makefile.include make-help + @echo 'environment' + @echo ' SEARX_URL = $(SEARX_URL)' + @echo ' GIT_URL = $(GIT_URL)' + @echo ' DOCS_URL = $(DOCS_URL)' @echo '' - @$(MAKE) -s -f utils/makefile.python python-help + @$(MAKE) -e -s make-help + +help-all: help-min + @echo '' + @$(MAKE) -e -s python-help + @echo '' + @$(MAKE) -e -s docs-help PHONY += install install: buildenv pyenvinstall @@ -44,7 +60,7 @@ PHONY += uninstall uninstall: pyenvuninstall PHONY += clean -clean: pyclean node.clean test.clean +clean: pyclean docs-clean node.clean test.clean $(call cmd,common_clean) PHONY += run @@ -61,14 +77,24 @@ run: buildenv pyenvinstall # docs # ---- +sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes + PHONY += docs -docs: buildenv pyenvinstall sphinx-doc +docs: sphinx-doc-prebuilds sphinx-doc $(call cmd,sphinx,html,docs,docs) PHONY += docs-live -docs-live: buildenv pyenvinstall sphinx-live +docs-live: sphinx-doc-prebuilds sphinx-live $(call cmd,sphinx_autobuild,html,docs,docs) +PHONY += prebuild-includes +prebuild-includes: + $(Q)mkdir -p $(DOCS_BUILD)/includes + $(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst + $(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst + $(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst + + $(GH_PAGES):: @echo "doc available at --> $(DOCS_URL)" @@ -94,12 +120,14 @@ useragents.update: pyenvinstall buildenv: $(Q)echo "build searx/brand.py" $(Q)echo "GIT_URL = '$(GIT_URL)'" > searx/brand.py + $(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'" >> searx/brand.py $(Q)echo "ISSUE_URL = 'https://github.com/asciimoo/searx/issues'" >> searx/brand.py $(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py $(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py $(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py $(Q)echo "build utils/brand.env" $(Q)echo "export GIT_URL='$(GIT_URL)'" > utils/brand.env + $(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'" >> utils/brand.env $(Q)echo "export ISSUE_URL='https://github.com/asciimoo/searx/issues'" >> utils/brand.env $(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env $(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env @@ -182,8 +210,7 @@ gecko.driver: # test # ---- -PHONY += test test.pylint test.pep8 test.unit test.coverage test.robot - +PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot ifeq ($(PY),2) @@ -191,6 +218,7 @@ test.pylint: @echo "LINT skip liniting py2" else # TODO: balance linting with pylint + test.pylint: pyenvinstall $(call cmd,pylint,\ searx/preferences.py \ @@ -202,6 +230,17 @@ endif # E402 module level import not at top of file # W503 line break before binary operator +# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported! +test.sh: + shellcheck -x -s bash utils/brand.env + shellcheck -x utils/lib.sh + shellcheck -x utils/filtron.sh + shellcheck -x utils/searx.sh + shellcheck -x utils/morty.sh + shellcheck -x utils/lxc.sh + shellcheck -x utils/lxc-searx.env + shellcheck -x .config.sh + test.pep8: pyenvinstall @echo "TEST pep8" $(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests diff --git a/docs/_themes/searx/static/searx.css b/docs/_themes/searx/static/searx.css index d6a664f0..0f2eff72 100644 --- a/docs/_themes/searx/static/searx.css +++ b/docs/_themes/searx/static/searx.css @@ -33,7 +33,7 @@ p.sidebar-title, .sidebar p { /* admonitions */ -div.admonition, div.topic { +div.admonition, div.topic, div.toctree-wrapper { background-color: #fafafa; margin: 8px 0px; padding: 1em; @@ -42,6 +42,16 @@ div.admonition, div.topic { border-right: none; border-bottom: none; border-left: 5pt solid #ccc; + list-style-type: disclosure-closed; +} + +div.toctree-wrapper p.caption { + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; + display: inline; } p.admonition-title:after { @@ -128,3 +138,32 @@ caption { caption-side: top; text-align: left; } + +/* bugs since sphinx 3.1 + +See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS: + +- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605 +- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972 + +*/ + +li > p:first-child { + margin-top: 0; +} + +li > p:last-child { + margin-bottom: 0; +} + +div.admonition dl { + margin-bottom: 0; +} + +div.sidebar { + clear: none; +} + +div.admonition, div.topic, pre { + clear: none; +} diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index a46c96de..5018225c 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -4,11 +4,11 @@ digraph G { edge [fontname="Sans"]; browser [label="Browser", shape=Mdiamond]; - rp [label="Reverse Proxy", href="url to configure reverse proxy"]; - filtron [label="Filtron", href="https://github.com/asciimoo/filtron"]; - morty [label="Morty", href="https://github.com/asciimoo/morty"]; + rp [label="Reverse Proxy", href="https://asciimoo.github.io/searx/utils/filtron.sh.html#public-reverse-proxy"]; + filtron [label="Filtron", href="https://asciimoo.github.io/searx/utils/filtron.sh.html"]; + morty [label="Morty", href="https://asciimoo.github.io/searx/utils/morty.sh.html"]; static [label="Static files", href="url to configure static files"]; - uwsgi [label="uwsgi", href="url to configure uwsgi"] + uwsgi [label="uwsgi", href="https://asciimoo.github.io/searx/utils/searx.sh.html"] searx1 [label="Searx #1"]; searx2 [label="Searx #2"]; searx3 [label="Searx #3"]; diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index 7064a294..464e765e 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -4,17 +4,21 @@ Architecture ============ -.. sidebar:: Needs work! +.. sidebar:: Further reading - This article needs some work / Searx is a collaborative effort. If you have - any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see - :ref:`how to contribute`. + - Reverse Proxy: :ref:`Apache ` & :ref:`nginx ` + - Filtron: :ref:`searx filtron` + - Morty: :ref:`searx morty` + - uWSGI: :ref:`searx uwsgi` + - Searx: :ref:`installation basic` Herein you will find some hints and suggestions about typical architectures of searx infrastructures. We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`. -It shows a *reference* setup for public searx instances. +It shows a *reference* setup for public searx instances which can build up and +maintained by the scripts from our :ref:`toolboxing`. .. _arch public: diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index 5260da03..a727d25b 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -9,8 +9,27 @@ Buildhosts If you have any contribution send us your :pull:`PR <../pulls>`, see :ref:`how to contribute`. +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + To get best results from build, its recommend to install additional packages -on build hosts. +on build hosts (see :ref:`searx.sh`).:: + + sudo -H ./utils/searx.sh install buildhost + +This will install packages needed by searx: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START distro-packages + :end-before: END distro-packages + +and packages needed to build docuemtation and run tests: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START build-packages + :end-before: END build-packages .. _docs build: @@ -35,8 +54,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF creation, its also needed for :ref:`math` when HTML output is build. To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered -as images (``sphinx.ext.imgmath`` extension). If your docs build (``make -docs``) shows warnings like this:: +as images (``sphinx.ext.imgmath`` extension). + +Here is the extract from the :origin:`docs/conf.py` file, setting math renderer +to ``imgmath``: + +.. literalinclude:: ../conf.py + :language: python + :start-after: # sphinx.ext.imgmath setup + :end-before: # sphinx.ext.imgmath setup END + +If your docs build (``make docs``) shows warnings like this:: WARNING: dot(1) not found, for better output quality install \ graphviz from http://www.graphviz.org @@ -47,8 +75,6 @@ docs``) shows warnings like this:: you need to install additional packages on your build host, to get better HTML output. -.. _system requirements: - .. tabs:: .. group-tab:: Ubuntu / debian @@ -92,12 +118,38 @@ For PDF output you also need: $ sudo dnf install \ texlive-collection-fontsrecommended texlive-collection-latex \ - dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts + dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \ + ImageMagick -.. _system requirements END: +.. _sh lint: -.. literalinclude:: ../conf.py - :language: python - :start-after: # sphinx.ext.imgmath setup - :end-before: # sphinx.ext.imgmath setup END +Lint shell scripts +================== +.. _ShellCheck: https://github.com/koalaman/shellcheck + +To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool. + +.. SNIP sh lint requirements + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo apt install shellcheck + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S shellcheck + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo dnf install ShellCheck + +.. SNAP sh lint requirements diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 07dcb9bc..93e430b1 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -1,18 +1,51 @@ + +.. _searx filtron: + ========================== How to protect an instance ========================== -Searx depens on external search services. To avoid the abuse of these services +.. sidebar:: further reading + + - :ref:`filtron.sh` + - :ref:`nginx searx site` + + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +.. _filtron: https://github.com/asciimoo/filtron + +Searx depends on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. -An application firewall, ``filtron`` solves exactly this problem. Information -on how to install it can be found at the `project page of filtron -`__. +An application firewall, filtron_ solves exactly this problem. Filtron is just +a middleware between your web server (nginx, apache, ...) and searx, we describe +such infratructures in chapter: :ref:`architecture`. +filtron & go +============ + +.. _Go: https://golang.org/ +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply +installed by ``go get`` package management (see `filtron README`_). If you use +filtron as middleware, a more isolated setup is recommended. To simplify such +an installation and the maintenance of, use our script :ref:`filtron.sh`. + +.. _Sample configuration of filtron: + Sample configuration of filtron =============================== +.. sidebar:: Tooling box + + - :origin:`/etc/filtron/rules.json ` + An example configuration can be find below. This configuration limits the access of: @@ -24,105 +57,104 @@ of: .. code:: json - [{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":"", - "limit":"", - "subrules":[ - { - "name":"roboagent limit", - "interval":"", - "limit":"", - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + [ + { + "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } + "interval": "" + "limit": "", + "subrules": [ + { + "name": "missing Accept-Language", + "filters": ["!Header:Accept-Language"], + "limit": "", + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "suspiciously Connection=close header", + "filters": ["Header:Connection=close"], + "limit": "", + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": "" + "limit": "", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "filters": [ + "Param:format=(csv|json|rss)" + ], + "interval": "" + "limit": "", + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "" + "limit": "", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } ] - }, - { - "name":"botlimit", - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":"", - "limit":"", - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":"", - "limit":"", - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":"", - "limit":"", - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - } - ] - }] + } + ] +.. _filtron route request: Route request through filtron ============================= +.. sidebar:: further reading + + - :ref:`filtron.sh overview` + - :ref:`installation nginx` + - :ref:`installation apache` + Filtron can be started using the following command: .. code:: sh @@ -136,13 +168,24 @@ Use it along with ``nginx`` with the following example configuration. .. code:: nginx - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_pass http://127.0.0.1:4004/; + # https://example.org/searx + + location /searx { + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; } + location /searx/static { + /usr/local/searx/searx-src/searx/static; + } + + Requests are coming from port 4004 going through filtron and then forwarded to -port 8888 where a searx is being run. +port 8888 where a searx is being run. For a complete setup see: :ref:`nginx +searx site`. diff --git a/docs/admin/index.rst b/docs/admin/index.rst index b3c7f511..c708c4ff 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -3,9 +3,16 @@ Administrator documentation =========================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents installation + installation-searx + installation-uwsgi + installation-nginx + installation-apache + installation-docker + update-searx settings api architecture diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst new file mode 100644 index 00000000..217e5771 --- /dev/null +++ b/docs/admin/installation-apache.rst @@ -0,0 +1,514 @@ +.. _installation apache: + +=================== +Install with apache +=================== + +.. _Apache: https://httpd.apache.org/ +.. _Apache Debian: + https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x): +.. _README.Debian: + https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian +.. _Apache Arch Linux: + https://wiki.archlinux.org/index.php/Apache_HTTP_Server +.. _Apache Fedora: + https://docs.fedoraproject.org/en-US/quick-docs/getting-started-with-apache-http-server/index.html +.. _Apache directives: + https://httpd.apache.org/docs/trunk/mod/directives.html +.. _Getting Started: + https://httpd.apache.org/docs/current/en/getting-started.html +.. _Terms Used to Describe Directives: + https://httpd.apache.org/docs/current/en/mod/directive-dict.html +.. _Configuration Files: + https://httpd.apache.org/docs/current/en/configuring.html +.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost +.. _LoadModule: + https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule +.. _DocumentRoot: + https://httpd.apache.org/docs/trunk/mod/core.html#documentroot +.. _Location: + https://httpd.apache.org/docs/trunk/mod/core.html#location +.. _uWSGI Apache support: + https://uwsgi-docs.readthedocs.io/en/latest/Apache.html +.. _mod_proxy_uwsgi: + https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi + +.. sidebar:: further read + + - `Apache Arch Linux`_ + - `Apache Debian`_ and `README.Debian`_ + - `Apache Fedora`_ + - `Apache directives`_ + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +---- + +**Install** :ref:`apache searx site` using :ref:`filtron.sh ` + +.. code:: bash + + $ sudo -H ./utils/filtron.sh apache install + +**Install** :ref:`apache searx site` using :ref:`morty.sh ` + +.. code:: bash + + $ sudo -H ./utils/morty.sh apache install + +---- + +The apache HTTP server +====================== + +If Apache_ is not installed, install it now. If apache_ is new to you, the +`Getting Started`_, `Configuration Files`_ and `Terms Used to Describe +Directives`_ documentation gives first orientation. There is also a list of +`Apache directives`_ *to keep in the pocket*. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install apache2 + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H pacman -S apache + sudo -H systemctl enable httpd + sudo -H systemctl start http + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + sudo -H dnf install httpd + sudo -H systemctl enable httpd + sudo -H systemctl start httpd + +Now at http://localhost you should see any kind of *Welcome* or *Test* page. +How this default intro site is configured, depends on the linux distribution +(compare `Apache directives`_). + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + less /etc/apache2/sites-enabled/000-default.conf + + In this file, there is a line setting the `DocumentRoot`_ directive: + + .. code:: apache + + DocumentRoot /var/www/html + + And the *welcome* page is the HTML file at ``/var/www/html/index.html``. + + .. group-tab:: Arch Linux + + .. code:: sh + + less /etc/httpd/conf/httpd.conf + + In this file, there is a line setting the `DocumentRoot`_ directive: + + .. code:: apache + + DocumentRoot "/srv/http" + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + + The *welcome* page of Arch Linux is a page showing directory located at + ``DocumentRoot``. This is *directory* page is generated by the Module + `mod_autoindex `_: + + .. code:: apache + + LoadModule autoindex_module modules/mod_autoindex.so + ... + Include conf/extra/httpd-autoindex.conf + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + less /etc/httpd/conf/httpd.conf + + In this file, there is a line setting the ``DocumentRoot`` directive: + + .. code:: apache + + DocumentRoot "/var/www/html" + ... + + AllowOverride None + # Allow open access: + Require all granted + + + On fresh installations, the ``/var/www`` is empty and the *default + welcome page* is shown, the configuration is located at:: + + less /etc/httpd/conf.d/welcome.conf + +.. _apache searx site: + +Apache Reverse Proxy +==================== + +.. sidebar:: public to the internet? + + If your searx instance is public, stop here and first install :ref:`filtron + reverse proxy ` and :ref:`result proxy morty `, see + :ref:`installation scripts`. If already done, follow setup: *searx via + filtron plus morty*. + +To setup a Apache revers proxy you have to enable the *headers* and *proxy* +modules and create a `Location`_ configuration for the searx site. In most +distributions you have to un-comment the lines in the main configuration file, +except in :ref:`The Debian Layout`. + +To pass the HTTP HOST header +With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the +proxied host. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + In the Apache setup, enable headers and proxy modules: + + .. code:: sh + + sudo -H a2enmod headers + sudo -H a2enmod proxy + sudo -H a2enmod proxy_http + + In :ref:`The Debian Layout` you create a ``searx.conf`` with the + ```` directive and save this file in the *sites + available* folder at ``/etc/apache2/sites-available``. To enable the + ``searx.conf`` use :man:`a2ensite`: + + .. code:: sh + + sudo -H a2ensite searx.conf + + .. group-tab:: Arch Linux + + In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy + modules (LoadModule_): + + .. code:: apache + + FIXME needs test + + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + + .. group-tab:: Fedora / RHEL + + In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy + modules (LoadModule_): + + .. code:: apache + + FIXME needs test + + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + +.. tabs:: + + .. group-tab:: searx via filtron plus morty + + Use this setup, if your instance is public to the internet, compare + figure: :ref:`architecture ` and :ref:`installation scripts`. + + 1. Configure a reverse proxy for :ref:`filtron `, listening on + *localhost 4004* (:ref:`filtron route request`): + + .. code:: apache + + + + # SetEnvIf Request_URI "/searx" dontlog + # CustomLog /dev/null combined env=dontlog + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass http://127.0.0.1:4004 + RequestHeader set X-Script-Name /searx + + + + 2. Configure reverse proxy for :ref:`morty `, listening on + *localhost 3000* + + .. code:: apache + + ProxyPreserveHost On + + + + # SetEnvIf Request_URI "/morty" dontlog + # CustomLog /dev/null combined env=dontlog + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://127.0.0.1:3000 + RequestHeader set X-Script-Name /morty + + + + Note that reverse proxy advised to be used in case of single-user or + low-traffic instances. For a fully result proxification add :ref:`morty's + ` **public URL** to your :origin:`searx/settings.yml`: + + .. code:: yaml + + result_proxy: + # replace example.org with your server's public name + url : https://example.org/morty + + server: + image_proxy : True + +uWSGI support +============= + +Be warned, with this setup, your instance isn't :ref:`protected `, nevertheless it is good enough for intranet usage. In modern Linux +distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache +package and you need to install only the :ref:`uWSGI ` package: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install uwsgi + + # Ubuntu =< 18.04 + sudo -H apt-get install libapache2-mod-proxy-uwsgi + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H pacman -S uwsgi + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + sudo -H dnf install uwsgi + +The next example shows a configuration using the `uWSGI Apache support`_ via +unix sockets and `mod_proxy_uwsgi`_. + +For socket communication, you have to activate ``socket = +/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888`` +configuration in your :ref:`uwsgi ini file `. If not +already exists, create a folder for the unix sockets, which can be used by the +searx account (see :ref:`create searx user`): + +.. code:: bash + + sudo -H mkdir -p /run/uwsgi/app/searx/ + sudo -H chown -R searx:searx /run/uwsgi/app/searx/ + +If the server is public; to limit access to your intranet replace ``Allow from +all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: apache + + LoadModule headers_module /usr/lib/apache2/mod_headers.so + LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so + LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so + + # SetEnvIf Request_URI /searx dontlog + # CustomLog /dev/null combined env=dontlog + + + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + + + + .. group-tab:: Arch Linux + + .. code:: apache + + FIXME needs test + + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so + + # SetEnvIf Request_URI /searx dontlog + # CustomLog /dev/null combined env=dontlog + + + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + + + + .. group-tab:: Fedora / RHEL + + .. code:: apache + + FIXME needs test + + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so + + + # SetEnvIf Request_URI /searx dontlog + # CustomLog /dev/null combined env=dontlog + + + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + + + + + + .. group-tab:: old mod_wsgi + + We show this only for historical reasons, DON'T USE `mod_uwsgi + `_. + ANYMORE! + + .. code:: apache + + + + # SetEnvIf Request_URI "/searx" dontlog + # CustomLog /dev/null combined env=dontlog + + + + Require all granted + + Options FollowSymLinks Indexes + SetHandler uwsgi-handler + uWSGISocket /run/uwsgi/app/searx/socket + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + + + + +.. _restart apache: + +Restart service +=============== + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart apache2 + sudo -H service uwsgi restart searx + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H systemctl restart httpd + sudo -H systemctl restart uwsgi@searx + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + sudo -H systemctl restart httpd + sudo -H touch /etc/uwsgi.d/searx.ini + + +disable logs +============ + +For better privacy you can disable Apache logs. In the examples above activate +one of the lines and `restart apache`_:: + + + # SetEnvIf Request_URI "/searx" dontlog + # CustomLog /dev/null combined env=dontlog + +The ``CustomLog`` directive disable logs for the whole (virtual) server, use it +when the URL of the service does not have a path component (``/searx``) / is +located at root (``/``). + +.. _The Debian Layout: + +The Debian Layout +================= + +Be aware that the Debian layout is quite different from the standard Apache +configuration. For details look at the README.Debian_ +(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on +Debian: + +* :man:`apache2ctl`: Apache HTTP server control interface +* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules +* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations +* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst new file mode 100644 index 00000000..340e6631 --- /dev/null +++ b/docs/admin/installation-docker.rst @@ -0,0 +1,28 @@ +.. _installation docker: + +=================== +Docker installation +=================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Make sure you have installed Docker. For instance, you can deploy searx like this: + +.. code:: sh + + docker pull wonderfall/searx + docker run -d --name searx -p $PORT:8888 wonderfall/searx + +Go to ``http://localhost:$PORT``. + +See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also +possible to build searx from the embedded Dockerfile. + +.. code:: sh + + git clone https://github.com/asciimoo/searx.git + cd searx + docker build -t whatever/searx . diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst new file mode 100644 index 00000000..5e32d968 --- /dev/null +++ b/docs/admin/installation-nginx.rst @@ -0,0 +1,381 @@ +.. _installation nginx: + +================== +Install with nginx +================== + +.. _nginx: + https://docs.nginx.com/nginx/admin-guide/ +.. _nginx server configuration: + https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers +.. _nginx beginners guide: + http://nginx.org/en/docs/beginners_guide.html +.. _Getting Started wiki: + https://www.nginx.com/resources/wiki/start/ +.. _uWSGI support from nginx: + https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html +.. _uwsgi_params: + https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#configuring-nginx +.. _SCRIPT_NAME: + https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name + +.. sidebar:: further reading + + - nginx_ + - `nginx beginners guide`_ + - `nginx server configuration`_ + - `Getting Started wiki`_ + - `uWSGI support from nginx`_ + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +---- + +**Install** :ref:`nginx searx site` using :ref:`filtron.sh ` + +.. code:: bash + + $ sudo -H ./utils/filtron.sh nginx install + +**Install** :ref:`nginx searx site` using :ref:`morty.sh ` + +.. code:: bash + + $ sudo -H ./utils/morty.sh nginx install + +---- + + +The nginx HTTP server +===================== + +If nginx_ is not installed (uwsgi will not work with the package nginx-light), +install it now. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install nginx + + .. group-tab:: Arch Linux + + .. code-block:: sh + + sudo -H pacman -S nginx-mainline + sudo -H systemctl enable nginx + sudo -H systemctl start nginx + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + sudo -H dnf install nginx + sudo -H systemctl enable nginx + sudo -H systemctl start nginx + +Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you +see a *Fedora Webserver - Test Page*. The test page comes from the default +`nginx server configuration`_. How this default intro site is configured, +depends on the linux distribution: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + less /etc/nginx/nginx.conf + + there is a line including site configurations from: + + .. code:: nginx + + include /etc/nginx/sites-enabled/*; + + .. group-tab:: Arch Linux + + .. code-block:: sh + + less /etc/nginx/nginx.conf + + in there is a configuration section named ``server``: + + .. code-block:: nginx + + server { + listen 80; + server_name localhost; + # ... + } + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + less /etc/nginx/nginx.conf + + there is a line including site configurations from: + + .. code:: nginx + + include /etc/nginx/conf.d/*.conf; + +.. _nginx searx site: + +A nginx searx site +================== + +.. sidebar:: public to the internet? + + If your searx instance is public, stop here and first install :ref:`filtron + reverse proxy ` and :ref:`result proxy morty `, see + :ref:`installation scripts`. If already done, follow setup: *searx via + filtron plus morty*. + +Now you have to create a configuration for the searx site. If nginx_ is new to +you, the `nginx beginners guide`_ is a good starting point and the `Getting +Started wiki`_ is always a good resource *to keep in the pocket*. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + Create configuration at ``/etc/nginx/sites-available/searx`` and place a + symlink to sites-enabled: + + .. code:: sh + + sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx + + .. group-tab:: Arch Linux + + In the ``/etc/nginx/nginx.conf`` file, replace the configuration section + named ``server``. + + .. group-tab:: Fedora / RHEL + + Create configuration at ``/etc/nginx/conf.d/searx`` and place a + symlink to sites-enabled: + +.. tabs:: + + .. group-tab:: searx via filtron plus morty + + Use this setup, if your instance is public to the internet, compare + figure: :ref:`architecture ` and :ref:`installation scripts`. + + 1. Configure a reverse proxy for :ref:`filtron `, listening on + *localhost 4004* (:ref:`filtron route request`): + + .. code:: nginx + + # https://example.org/searx + + location /searx { + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; + } + + location /searx/static { + /usr/local/searx/searx-src/searx/static; + } + + + 2. Configure reverse proxy for :ref:`morty `, listening on + *localhost 3000*: + + .. code:: nginx + + # https://example.org/morty + + location /morty { + proxy_pass http://127.0.0.1:3000/; + + proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + } + + Note that reverse proxy advised to be used in case of single-user or + low-traffic instances. For a fully result proxification add :ref:`morty's + ` **public URL** to your :origin:`searx/settings.yml`: + + .. code:: yaml + + result_proxy: + # replace example.org with your server's public name + url : https://example.org/morty + + server: + image_proxy : True + + + .. group-tab:: proxy or uWSGI + + Be warned, with this setup, your instance isn't :ref:`protected `. Nevertheless it is good enough for intranet usage and it is a + excellent example of; *how different services can be set up*. The next + example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI + application `, listening on ``http = + 127.0.0.1:8888``. + + .. code:: nginx + + # https://hostname.local/ + + location / { + proxy_pass http://127.0.0.1:8888; + + proxy_set_header Host $host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_buffering off; + } + + Alternatively you can use the `uWSGI support from nginx`_ via unix + sockets. For socket communication, you have to activate ``socket = + /run/uwsgi/app/searx/socket`` and comment out the ``http = + 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file `. + + The example shows a nginx virtual ``server`` configuration, listening on + port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at + location ``/`` by importing the `uwsgi_params`_ and passing requests to + the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the + :ref:`searx-src clone ` and wraps directly the + :origin:`searx/static/` content at ``location /static``. + + .. code:: nginx + + server { + # replace hostname.local with your server's name + server_name hostname.local; + + listen 80; + listen [::]:80; + + location / { + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/app/searx/socket; + } + + root /usr/local/searx/searx-src/searx; + location /static { } + } + + If not already exists, create a folder for the unix sockets, which can be + used by the searx account: + + .. code:: bash + + mkdir -p /run/uwsgi/app/searx/ + sudo -H chown -R searx:searx /run/uwsgi/app/searx/ + + .. group-tab:: \.\. at subdir URL + + Be warned, with these setups, your instance isn't :ref:`protected `. The examples are just here to demonstrate how to export the + searx application from a subdirectory URL ``https://example.org/searx/``. + + .. code:: nginx + + # https://hostname.local/searx + + location /searx { + proxy_pass http://127.0.0.1:8888; + + proxy_set_header Host $host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; + proxy_buffering off; + } + + location /searx/static { + alias /usr/local/searx/searx-src/searx/static; + } + + The ``X-Script-Name /searx`` is needed by the searx implementation to + calculate relative URLs correct. The next example shows a uWSGI + configuration. Since there are no HTTP headers in a (u)WSGI protocol, the + value is shipped via the SCRIPT_NAME_ in the WSGI environment. + + .. code:: nginx + + # https://hostname.local/searx + + location /searx { + uwsgi_param SCRIPT_NAME /searx; + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/app/searx/socket; + } + + location /searx/static { + alias /usr/local/searx/searx-src/searx; + } + + For searx to work correctly the ``base_url`` must be set in the + :origin:`searx/settings.yml`. + + .. code:: yaml + + server: + # replace example.org with your server's public name + base_url : https://example.org/searx/ + + +Restart service: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H service uwsgi restart searx + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H systemctl restart uwsgi@searx + + .. group-tab:: Fedora + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H touch /etc/uwsgi.d/searx.ini + + +Disable logs +============ + +For better privacy you can disable nginx logs in ``/etc/nginx/nginx.conf``. + +.. code:: nginx + + http { + # ... + access_log /dev/null; + error_log /dev/null; + # ... + } diff --git a/docs/admin/installation-searx.rst b/docs/admin/installation-searx.rst new file mode 100644 index 00000000..f1d48602 --- /dev/null +++ b/docs/admin/installation-searx.rst @@ -0,0 +1,92 @@ +.. _installation basic: + +========================= +Step by step installation +========================= + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Step by step installation with virtualenv. For Ubuntu, be sure to have enable +universe repository. + +.. _install packages: + +Install packages +================ + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START distro-packages + :end-before: END distro-packages + +.. hint:: + + This installs also the packages needed by :ref:`searx uwsgi` + +.. _create searx user: + +Create user +=========== + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START create user + :end-before: END create user + +.. _searx-src: + +install searx & dependencies +============================ + +Start a interactive shell from new created user and clone searx: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START clone searx + :end-before: END clone searx + +In the same shell create *virtualenv*: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START create virtualenv + :end-before: END create virtualenv + +To install searx's dependencies, exit the searx *bash* session you opened above +and restart a new. Before install, first check if your *virualenv* was sourced +from the login (*~/.profile*): + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START manage.sh update_packages + :end-before: END manage.sh update_packages + +.. tip:: + + Open a second terminal for the configuration tasks and left the ``(searx)$`` + terminal open for the tasks below. + +Configuration +============== + +Create a copy of the :origin:`searx/settings.yml` configuration file in system's +*/etc* folder. Configure like shown below -- replace ``searx@\$(uname -n)`` with +a name of your choice -- *and/or* edit ``/etc/searx/settings.yml`` if necessary. + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx config + :end-before: END searx config + +Check +===== + +To check your searx setup, optional enable debugging and start the *webapp*. +Searx looks at the exported environment ``$SEARX_SETTINGS_PATH`` for a +configuration file. + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START check searx installation + :end-before: END check searx installation + +If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the +debug option in ``settings.yml``. You can now exit searx user bash (enter exit +command twice). At this point searx is not demonized; uwsgi allows this. + diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst new file mode 100644 index 00000000..ac4c463b --- /dev/null +++ b/docs/admin/installation-uwsgi.rst @@ -0,0 +1,149 @@ +.. _searx uwsgi: + +===== +uwsgi +===== + +.. sidebar:: further reading + + - `systemd.unit`_ + - `uWSGI Emperor`_ + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + + +.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html +.. _One service per app in systemd: + https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd +.. _uWSGI Emperor: + https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html +.. _uwsgi ini file: + https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files +.. _systemd unit template: + http://0pointer.de/blog/projects/instances.html + + +Origin uWSGI +============ + +How uWSGI is implemented by distributors is different. uWSGI itself +recommend two methods + +`systemd.unit`_ template files as described here `One service per app in systemd`_. + + There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app + placed at dedicated locations. Take archlinux and a searx.ini as example:: + + unit template --> /usr/lib/systemd/system/uwsgi@.service + uwsgi ini files --> /etc/uwsgi/searx.ini + + The searx app can be maintained as know from common systemd units:: + + systemctl enable uwsgi@searx + systemctl start uwsgi@searx + systemctl restart uwsgi@searx + systemctl stop uwsgi@searx + +The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps. + + The Emperor mode is a special uWSGI instance that will monitor specific + events. The Emperor mode (service) is started by a (common, not template) + systemd unit. The Emperor service will scan specific directories for `uwsgi + ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the + timestamp is modified, a corresponding action takes place: a new uWSGI + instance is started, reload or stopped. Take Fedora and a searx.ini as + example:: + + to start a new searx instance create --> /etc/uwsgi.d/searx.ini + to reload the instance edit timestamp --> touch /etc/uwsgi.d/searx.ini + to stop instance remove ini --> rm /etc/uwsgi.d/searx.ini + +Distributors +============ + +The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors +mostly offer their users, even if they differ in the way they implement both +modes and their defaults. Another point they might differ is the packaging of +plugins (if so, compare :ref:`install packages`) and what the default python +interpreter is (python2 vs. python3). + +Fedora starts a Emperor by default, while archlinux does not start any uwsgi +service by default. Worth to know; debian (ubuntu) follow a complete different +approach. *debian*: your are familiar with the apache infrastructure? .. they +do similar for the uWSGI infrastructure (with less comfort), the folders are:: + + /etc/uwsgi/apps-available/ + /etc/uwsgi/apps-enabled/ + +The `uwsgi ini file`_ is enabled by a symbolic link:: + + ln -s /etc/uwsgi/apps-available/searx.ini /etc/uwsgi/apps-enabled/ + +From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You +could control specific instance(s) by issuing:: + + service uwsgi ... + + sudo -H service uwsgi start searx + sudo -H service uwsgi stop searx + +My experience is, that this command is a bit buggy. + +.. _uwsgi configuration: + +Alltogether +=========== + +Create the configuration ini-file according to your distribution (see below) and +restart the uwsgi application. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx uwsgi-description ubuntu-20.04 + :end-before: END searx uwsgi-description ubuntu-20.04 + + + .. group-tab:: Arch Linux + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx uwsgi-description arch + :end-before: END searx uwsgi-description arch + + + .. group-tab:: Fedora / RHEL + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx uwsgi-description fedora + :end-before: END searx uwsgi-description fedora + + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :code: ini + :start-after: START searx uwsgi-appini ubuntu-20.04 + :end-before: END searx uwsgi-appini ubuntu-20.04 + + .. group-tab:: Arch Linux + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :code: ini + :start-after: START searx uwsgi-appini arch + :end-before: END searx uwsgi-appini arch + + .. group-tab:: Fedora / RHEL + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :code: ini + :start-after: START searx uwsgi-appini fedora + :end-before: END searx uwsgi-appini fedora + + diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 15800fc0..167c300f 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -4,346 +4,63 @@ Installation ============ -.. contents:: - :depth: 3 +*You're spoilt for choice*, choose your preferred method of installation. -Basic installation -================== +- :ref:`installation docker` +- :ref:`installation scripts` +- :ref:`installation basic` -Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure -to have enable universe repository. +The :ref:`installation basic` is good enough for intranet usage and it is a +excellent illustration of *how a searx instance is build up*. If you place your +instance public to the internet you should really consider to install a +:ref:`filtron reverse proxy ` and for privacy a :ref:`result proxy +` is mandatory. -Install packages: +Therefore, if you do not have any special preferences, its recommend to use the +:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling +box ` as described below. -.. code:: sh +.. _installation scripts: - $ sudo -H apt-get install \ - git build-essential libxslt-dev \ - python-dev python-virtualenv python-babel \ - zlib1g-dev libffi-dev libssl-dev +Installation scripts +==================== -Install searx: +.. sidebar:: Update OS first! -.. code:: sh + To avoid unwanted side effects, update your OS before installing searx. - cd /usr/local - sudo -H git clone https://github.com/asciimoo/searx.git - sudo -H useradd searx -d /usr/local/searx - sudo -H chown searx:searx -R /usr/local/searx +The following will install a setup as shown in :ref:`architecture`. First you +need to get a clone. The clone is only needed for the installation procedure +and some maintenance tasks (alternatively you can create your own fork). -Install dependencies in a virtualenv: +.. code:: bash -.. code:: sh + $ cd ~/Downloads + $ git clone https://github.com/asciimoo/searx searx + $ cd searx - cd /usr/local/searx - sudo -H -u searx -i +**Install** :ref:`searx service ` -.. code:: sh +This installs searx as described in :ref:`installation basic`. - (searx)$ virtualenv searx-ve - (searx)$ . ./searx-ve/bin/activate - (searx)$ ./manage.sh update_packages +.. code:: bash -Configuration -============== + $ sudo -H ./utils/searx.sh install all -.. code:: sh +**Install** :ref:`filtron reverse proxy ` - sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml +.. code:: bash -Edit searx/settings.yml if necessary. + $ sudo -H ./utils/filtron.sh install all -Check -===== +**Install** :ref:`result proxy ` -Start searx: +.. code:: bash -.. code:: sh + $ sudo -H ./utils/morty.sh install all - python searx/webapp.py +If all services are running fine, you can add it to your HTTP server: -Go to http://localhost:8888 +- :ref:`installation apache` +- :ref:`installation nginx` -If everything works fine, disable the debug option in settings.yml: - -.. code:: sh - - sed -i -e "s/debug : True/debug : False/g" searx/settings.yml - -At this point searx is not demonized ; uwsgi allows this. - -You can exit the virtualenv and the searx user bash (enter exit command -twice). - -uwsgi -===== - -Install packages: - -.. code:: sh - - sudo -H apt-get install \ - uwsgi uwsgi-plugin-python - -Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this -content: - -.. code:: ini - - [uwsgi] - # Who will run the code - uid = searx - gid = searx - - # disable logging for privacy - disable-logging = true - - # Number of workers (usually CPU count) - workers = 4 - - # The right granted on the created socket - chmod-socket = 666 - - # Plugin to use and interpretor config - single-interpreter = true - master = true - plugin = python - lazy-apps = true - enable-threads = true - - # Module to import - module = searx.webapp - - # Support running the module from a webserver subdirectory. - route-run = fixpathinfo: - - # Virtualenv and python path - virtualenv = /usr/local/searx/searx-ve/ - pythonpath = /usr/local/searx/ - chdir = /usr/local/searx/searx/ - -Activate the uwsgi application and restart: - -.. code:: sh - - cd /etc/uwsgi/apps-enabled - ln -s ../apps-available/searx.ini - /etc/init.d/uwsgi restart - -Web server -========== - -with nginx ----------- - -If nginx is not installed (uwsgi will not work with the package -nginx-light): - -.. code:: sh - - sudo -H apt-get install nginx - -Hosted at / -~~~~~~~~~~~ - -Create the configuration file ``/etc/nginx/sites-available/searx`` with this -content: - -.. code:: nginx - - server { - listen 80; - server_name searx.example.com; - root /usr/local/searx/searx; - - location /static { - } - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - } - -Create a symlink to sites-enabled: - -.. code:: sh - - sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - sudo -H service uwsgi restart - -from subdirectory URL (/searx) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Add this configuration in the server config file -``/etc/nginx/sites-enabled/default``: - -.. code:: nginx - - location /searx/static { - alias /usr/local/searx/searx/static; - } - - location /searx { - uwsgi_param SCRIPT_NAME /searx; - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - - -**OR** using reverse proxy (Please, note that reverse proxy advised to be used -in case of single-user or low-traffic instances.) - -.. code:: nginx - - location /searx/static { - alias /usr/local/searx/searx/static; - } - - location /searx { - proxy_pass http://127.0.0.1:8888; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - proxy_buffering off; - } - - -Enable ``base_url`` in ``searx/settings.yml`` - -.. code:: yaml - - base_url : http://your.domain.tld/searx/ - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - sudo -H service uwsgi restart - -disable logs -^^^^^^^^^^^^ - -for better privacy you can disable nginx logs about searx. - -how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default`` -add: - -.. code:: nginx - - access_log /dev/null; - error_log /dev/null; - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - -with apache ------------ - -Add wsgi mod: - -.. code:: sh - - sudo -H apt-get install libapache2-mod-uwsgi - sudo -H a2enmod uwsgi - -Add this configuration in the file ``/etc/apache2/apache2.conf``: - -.. code:: apache - - - Options FollowSymLinks Indexes - SetHandler uwsgi-handler - uWSGISocket /run/uwsgi/app/searx/socket - - -Note that if your instance of searx is not at the root, you should change -```` by the location of your instance, like ````. - -Restart Apache: - -.. code:: sh - - sudo -H /etc/init.d/apache2 restart - -disable logs -~~~~~~~~~~~~ - -For better privacy you can disable Apache logs. - -.. warning:: - - You can only disable logs for the whole (virtual) server not for a specific - path. - -Go back to ``/etc/apache2/apache2.conf`` and above ```` add: - -.. code:: apache - - CustomLog /dev/null combined - -Restart Apache: - -.. code:: sh - - sudo -H /etc/init.d/apache2 restart - -How to update -============= - -.. code:: sh - - cd /usr/local/searx - sudo -H -u searx -i - -.. code:: sh - - (searx)$ . ./searx-ve/bin/activate - (searx)$ git stash - (searx)$ git pull origin master - (searx)$ git stash apply - (searx)$ ./manage.sh update_packages - -.. code:: sh - - sudo -H service uwsgi restart - -Docker -====== - -Make sure you have installed Docker. For instance, you can deploy searx like this: - -.. code:: sh - - docker pull wonderfall/searx - docker run -d --name searx -p $PORT:8888 wonderfall/searx - -Go to ``http://localhost:$PORT``. - -See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also -possible to build searx from the embedded Dockerfile. - -.. code:: sh - - git clone https://github.com/asciimoo/searx.git - cd searx - docker build -t whatever/searx . - -References -========== - -* https://about.okhin.fr/posts/Searx/ with some additions - -* How to: `Setup searx in a couple of hours with a free SSL certificate - `__ diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst index 7d7b3449..2858fde3 100644 --- a/docs/admin/morty.rst +++ b/docs/admin/morty.rst @@ -1,7 +1,14 @@ + +.. _searx morty: + ========================= How to setup result proxy ========================= +.. sidebar:: further reading + + - :ref:`morty.sh` + .. _morty: https://github.com/asciimoo/morty .. _morty's README: https://github.com/asciimoo/morty diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index 2bfbae35..8b1cb8dc 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -4,11 +4,17 @@ ``settings.yml`` ================ +This page describe the options possibilities of the :origin:`searx/settings.yml` +file. + .. sidebar:: Further reading .. - :ref:`search API` -This page describe the options possibilities of the settings.yml file. +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry .. _settings global: diff --git a/docs/admin/update-searx.rst b/docs/admin/update-searx.rst new file mode 100644 index 00000000..c74c9568 --- /dev/null +++ b/docs/admin/update-searx.rst @@ -0,0 +1,23 @@ +.. _update searx: + +============= +How to update +============= + +.. code:: sh + + sudo -H -u searx -i + (searx)$ git stash + (searx)$ git pull origin master + (searx)$ git stash apply + (searx)$ ./manage.sh update_packages + +Restart uwsgi: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart uwsgi diff --git a/docs/blog/index.rst b/docs/blog/index.rst index 94847ad6..5a8135fd 100644 --- a/docs/blog/index.rst +++ b/docs/blog/index.rst @@ -3,7 +3,8 @@ Blog ==== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents python3 admin diff --git a/docs/build-templates/filtron.rst b/docs/build-templates/filtron.rst new file mode 100644 index 00000000..83a45182 --- /dev/null +++ b/docs/build-templates/filtron.rst @@ -0,0 +1,52 @@ +.. START create user + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H useradd --shell /bin/bash --system \\ + --home-dir "$SERVICE_HOME" \\ + --comment "Privacy-respecting metasearch engine" $SERVICE_USER + + $ sudo -H mkdir "$SERVICE_HOME" + $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" + +.. END create user + +.. START install go + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ cat > "$GO_ENV" <> ~/.profile + (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local + (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\ + "${GO_PKG_URL}" + (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}" + (${SERVICE_USER}) $ which go + ${SERVICE_HOME}/local/go/bin/go + +.. END install go + +.. START install filtron + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ sudo -i -u "${SERVICE_USER}" + (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/filtron + +.. END install filtron diff --git a/docs/build-templates/morty.rst b/docs/build-templates/morty.rst new file mode 100644 index 00000000..4a5d1f27 --- /dev/null +++ b/docs/build-templates/morty.rst @@ -0,0 +1,52 @@ +.. START create user + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H useradd --shell /bin/bash --system \\ + --home-dir "$SERVICE_HOME" \\ + --comment "Privacy-respecting metasearch engine" $SERVICE_USER + + $ sudo -H mkdir "$SERVICE_HOME" + $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" + +.. END create user + +.. START install go + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ cat > "$GO_ENV" <> ~/.profile + (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local + (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\ + "${GO_PKG_URL}" + (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}" + (${SERVICE_USER}) $ which go + ${SERVICE_HOME}/local/go/bin/go + +.. END install go + +.. START install morty + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ sudo -i -u "${SERVICE_USER}" + (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty + +.. END install morty diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searx.rst new file mode 100644 index 00000000..5cb70e95 --- /dev/null +++ b/docs/build-templates/searx.rst @@ -0,0 +1,192 @@ +.. template evaluated by: ./utils/searx.sh docs +.. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$ + +.. START distro-packages + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo -H apt-get install -y \\ +${debian} + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo -H pacman -S --noconfirm \\ +${arch} + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo -H dnf install -y \\ +${fedora} + +.. END distro-packages + +.. START build-packages + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo -H apt-get install -y \\ +${debian_build} + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo -H pacman -S --noconfirm \\ +${arch_build} + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo -H dnf install -y \\ +${fedora_build} + +.. END build-packages + +.. START create user + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H useradd --shell /bin/bash --system \\ + --home-dir "$SERVICE_HOME" \\ + --comment "Privacy-respecting metasearch engine" $SERVICE_USER + + $ sudo -H mkdir "$SERVICE_HOME" + $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" + +.. END create user + +.. START clone searx + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H -u ${SERVICE_USER} -i + (${SERVICE_USER})$ git clone "https://github.com/asciimoo/searx.git" "$SEARX_SRC" + +.. END clone searx + +.. START create virtualenv + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + (${SERVICE_USER})$ python3 -m venv "${SEARX_PYENV}" + (${SERVICE_USER})$ echo ". ${SEARX_PYENV}/bin/activate" >> "$SERVICE_HOME/.profile" + +.. END create virtualenv + +.. START manage.sh update_packages + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H -u ${SERVICE_USER} -i + + (${SERVICE_USER})$ command -v python && python --version + $SEARX_PYENV/bin/python + Python 3.8.1 + + # update pip's boilerplate .. + pip install -U pip + pip install -U setuptools + pip install -U wheel + + # jump to searx's working tree and install searx into virtualenv + (${SERVICE_USER})$ cd "$SEARX_SRC" + (${SERVICE_USER})$ pip install -e . + + +.. END manage.sh update_packages + +.. START searx config + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H cp "$SEARX_SRC/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + $ sudo -H sed -i -e "s/ultrasecretkey/\\$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" + $ sudo -H sed -i -e "s/{instance_name}/searx@\\$(uname -n)/g" "$SEARX_SETTINGS_PATH" + +.. END searx config + +.. START check searx installation + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + # enable debug .. + $ sudo -H sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" + + # start webapp + $ sudo -H -u ${SERVICE_USER} -i + (${SERVICE_USER})$ cd ${SEARX_SRC} + (${SERVICE_USER})$ export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}" + (${SERVICE_USER})$ python searx/webapp.py + + # disable debug + $ sudo -H sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" + +Open WEB browser and visit http://$SEARX_INTERNAL_URL . If you are inside a +container or in a script, test with curl: + +.. tabs:: + + .. group-tab:: WEB browser + + .. code-block:: sh + + $ xgd-open http://$SEARX_INTERNAL_URL + + .. group-tab:: curl + + .. code-block:: none + + $ curl --location --verbose --head --insecure $SEARX_INTERNAL_URL + + * Trying 127.0.0.1:8888... + * TCP_NODELAY set + * Connected to 127.0.0.1 (127.0.0.1) port 8888 (#0) + > HEAD / HTTP/1.1 + > Host: 127.0.0.1:8888 + > User-Agent: curl/7.68.0 + > Accept: */* + > + * Mark bundle as not supporting multiuse + * HTTP 1.0, assume close after body + < HTTP/1.0 200 OK + HTTP/1.0 200 OK + ... + +.. END check searx installation diff --git a/docs/conf.py b/docs/conf.py index 03e977ab..1f79e4c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import sys, os +from sphinx_build_tools import load_sphinx_config from searx.version import VERSION_STRING from pallets_sphinx_themes import ProjectLink from searx.brand import GIT_URL +GIT_BRANCH = os.environ.get("GIT_BRANCH", "master") from searx.brand import SEARX_URL from searx.brand import DOCS_URL @@ -22,6 +24,8 @@ master_doc = "index" source_suffix = '.rst' numfig = True +exclude_patterns = ['build-templates/*.rst'] + from searx import webapp jinja_contexts = { 'webapp': dict(**webapp.__dict__) @@ -35,7 +39,7 @@ extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ') extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ') # links to custom brand -extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://') +extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://') extlinks['patch'] = (GIT_URL + '/commit/%s', '#') extlinks['search'] = (SEARX_URL + '/%s', '#') extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ') @@ -61,6 +65,8 @@ extensions = [ "pallets_sphinx_themes", "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja + "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput + 'linuxdoc.kernel_include', # Implementation of the 'kernel-include' reST-directive. 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive. 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling. "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs @@ -112,3 +118,9 @@ html_show_sourcelink = False latex_documents = [ (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual") ] + +# ------------------------------------------------------------------------------ +# Since loadConfig overwrites settings from the global namespace, it has to be +# the last statement in the conf.py file +# ------------------------------------------------------------------------------ +load_sphinx_config(globals()) diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 459dfb44..f9e9569f 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -4,6 +4,11 @@ How to contribute ================= +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + Prime directives: Privacy, Hackability ====================================== diff --git a/docs/dev/index.rst b/docs/dev/index.rst index cb913a82..ba0a25a9 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -3,7 +3,8 @@ Developer documentation ======================= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents quickstart contribution_guide diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index f5957001..62cd0a98 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -11,23 +11,17 @@ Makefile Targets Before looking deeper at the targets, first read about :ref:`makefile setup` and :ref:`make pyenv`. + To install system requirements follow :ref:`buildhosts`. + With the aim to simplify development cycles, started with :pull:`1756` a ``Makefile`` based boilerplate was added. If you are not familiar with Makefiles, we recommend to read gnu-make_ introduction. The usage is simple, just type ``make {target-name}`` to *build* a target. -Calling the ``help`` target gives a first overview:: +Calling the ``help`` target gives a first overview (``make help``): + +.. program-output:: bash -c "cd ..; make --no-print-directory help" - $ make help - test - run developer tests - docs - build documentation - docs-live - autobuild HTML documentation while editing - run - run developer instance - install - developer install (./local) - uninstall - uninstall (./local) - gh-pages - build docs & deploy on gh-pages branch - clean - drop builds and environments - ... .. contents:: Contents :depth: 2 @@ -37,27 +31,33 @@ Calling the ``help`` target gives a first overview:: .. _makefile setup: -Setup -===== +Makefile setup +============== .. _git stash: https://git-scm.com/docs/git-stash -The main setup is done in the :origin:`Makefile`:: - - export GIT_URL=https://github.com/asciimoo/searx - export SEARX_URL=https://searx.me - export DOCS_URL=https://asciimoo.github.io/searx - .. sidebar:: fork & upstream Commit changes in your (local) branch, fork or whatever, but do not push them upstream / `git stash`_ is your friend. -:GIT_URL: Changes this, to point to your searx fork. +The main setup is done in the :origin:`Makefile`. -:SEARX_URL: Changes this, to point to your searx instance. +.. literalinclude:: ../../Makefile + :start-after: START Makefile setup + :end-before: END Makefile setup -:DOCS_URL: If you host your own (branded) documentation, change this URL. +:GIT_URL: Changes this, to point to your searx fork. +:GIT_BRANCH: Changes this, to point to your searx branch. +:SEARX_URL: Changes this, to point to your searx instance. +:DOCS_URL: If you host your own (*brand*) documentation, change this URL. + +If you change any of this build environment variables, you have to run ``make +buildenv``:: + + $ make buildenv + build searx/brand.py + build utils/brand.env .. _make pyenv: @@ -170,7 +170,7 @@ e.g.: .. code:: sh - $ make test.pep8 test.unit + $ make test.pep8 test.unit test.sh . ./local/py3/bin/activate; ./manage.sh pep8_check [!] Running pep8 check . ./local/py3/bin/activate; ./manage.sh unit_tests diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index a786eda6..a4fc897c 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -27,7 +27,7 @@ searx-ve virtualenv and install the required packages using ``manage.sh``. cd ~/myprojects git clone https://github.com/asciimoo/searx.git cd searx - virtualenv searx-ve + python3 -m venv searx-ve . ./searx-ve/bin/activate ./manage.sh update_dev_packages diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 4835cbbc..906a0e9a 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -325,8 +325,9 @@ Literal blocks The simplest form of :duref:`literal-blocks` is a indented block introduced by two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST -code` directive. To include literals from external files use directive -:dudir:`literalinclude`. +code` directive. To include literals from external files use +:rst:dir:`literalinclude` or :ref:`kernel-include ` +directive (latter one expands environment variables in the path name). .. _reST literal: @@ -1312,9 +1313,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from .. literalinclude:: ../admin/buildhosts.rst :language: reST - :start-after: .. _system requirements: - :end-before: .. _system requirements END: - + :start-after: .. SNIP sh lint requirements + :end-before: .. SNAP sh lint requirements .. _math: diff --git a/docs/index.rst b/docs/index.rst index b62f4dc5..9e590867 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,14 @@ Welcome to searx ================ -Search without being tracked. + *Search without being tracked.* + +Searx is a free internet metasearch engine which aggregates results from more +than 70 search services. Users are neither tracked nor profiled. Additionally, +searx can be used over Tor for online anonymity. + +Get started with searx by using one of the Searx-instances_. If you don't trust +anyone, you can set up your own, see :ref:`installation`. .. sidebar:: Features @@ -16,19 +23,14 @@ Search without being tracked. - Hosted by organizations, such as *La Quadrature du Net*, which promote digital rights -Searx is a free internet metasearch engine which aggregates results from more -than 70 search services. Users are neither tracked nor profiled. Additionally, -searx can be used over Tor for online anonymity. - -Get started with searx by using one of the Searx-instances_. If you don't trust -anyone, you can set up your own, see :ref:`installation`. - .. toctree:: :maxdepth: 2 + :caption: Contents user/index admin/index dev/index + utils/index blog/index .. _Searx-instances: https://searx.space diff --git a/docs/user/conf.py b/docs/user/conf.py new file mode 100644 index 00000000..53ade4b6 --- /dev/null +++ b/docs/user/conf.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8; mode: python -*- +"""Configuration for the Searx user handbook +""" +project = 'Searx User-HB' +version = release = VERSION_STRING + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index' # startdocname + , 'searx-user-hb.tex' # targetname + , '' # take title from .rst + , author # author + , 'howto' # documentclass + , False # toctree_only + ), +] + diff --git a/docs/user/index.rst b/docs/user/index.rst index b13aca21..96d11bf8 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -3,7 +3,8 @@ User documentation ================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents search_syntax own-instance diff --git a/docs/user/own-instance.rst b/docs/user/own-instance.rst index a2f73656..4876abe9 100644 --- a/docs/user/own-instance.rst +++ b/docs/user/own-instance.rst @@ -2,8 +2,10 @@ Why use a private instance? =========================== -"Is it worth to run my own instance?" is a common question among searx users. -Before answering this question, see what options a searx user has. + *"Is it worth to run my own instance?"* + +\.\. is a common question among searx users. Before answering this question, +see what options a searx user has. Public instances are open to everyone who has access to its URL. Usually, these are operated by unknown parties (from the users' point of view). Private diff --git a/docs/user/public_instances.rst b/docs/user/public_instances.rst deleted file mode 100644 index f58ab7a0..00000000 --- a/docs/user/public_instances.rst +++ /dev/null @@ -1,3 +0,0 @@ -:orphan: - -This page page has been moved to `searx.space `__ diff --git a/docs/utils/filtron.sh.rst b/docs/utils/filtron.sh.rst new file mode 100644 index 00000000..86e3fa88 --- /dev/null +++ b/docs/utils/filtron.sh.rst @@ -0,0 +1,80 @@ + +.. _filtron.sh: + +==================== +``utils/filtron.sh`` +==================== + +.. sidebar:: further reading + + - :ref:`searx filtron` + - :ref:`architecture` + - :ref:`installation` (:ref:`nginx ` & :ref:`apache + `) + +.. _Go: https://golang.org/ +.. _filtron: https://github.com/asciimoo/filtron +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +To simplify installation and maintenance of a filtron instance you can use the +script :origin:`utils/filtron.sh`. In most cases you will install filtron_ +simply by running the command: + +.. code:: bash + + sudo -H ./utils/filtron.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_ +into this user account: + +#. Create a separated user account (``filtron``). +#. Download and install Go_ binary in user's $HOME (``~filtron``). +#. Install filtron with the package management from Go_ (``go get -v -u + github.com/asciimoo/filtron``) +#. Setup a proper rule configuration :origin:`[ref] + ` (``/etc/filtron/rules.json``). +#. Setup a systemd service unit :origin:`[ref] + ` + (``/lib/systemd/system/filtron.service``). + + +Create user +=========== + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START create user + :end-before: END create user + + +Install go +========== + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START install go + :end-before: END install go + + +Install filtron +=============== + +Install :origin:`rules.json ` at +``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and +install filtron software and systemd unit: + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START install filtron + :end-before: END install filtron + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START install systemd unit + :end-before: END install systemd unit + +.. _filtron.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/filtron.sh --help diff --git a/docs/utils/index.rst b/docs/utils/index.rst new file mode 100644 index 00000000..13914af2 --- /dev/null +++ b/docs/utils/index.rst @@ -0,0 +1,53 @@ +.. _searx_utils: +.. _toolboxing: + +======================= +Tooling box ``utils/*`` +======================= + +In the folder :origin:`utils/` we maintain some tools useful for admins and +developers. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + searx.sh + filtron.sh + morty.sh + lxc.sh + +.. _toolboxing common: + +Common commands & environment +============================= + +Scripts to maintain services often dispose of common commands and environments. + +``shell`` : command + Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for + troubleshooting. + +``inspect service`` : command + Shows status and log of the service, most often you have a option to enable + more verbose debug logs. Very helpful for debugging, but be careful not to + enable debugging in a production environment! + +``FORCE_TIMEOUT`` : environment + Sets timeout for interactive prompts. If you want to run a script in batch + job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a + reverse proxy for filtron on all containers of the :ref:`searx suite + ` use :: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install + +.. _toolboxing setup: + +Tooling box setup +================= + +The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile +setup`). + +.. literalinclude:: ../../.config.sh + :language: bash diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst new file mode 100644 index 00000000..ae141287 --- /dev/null +++ b/docs/utils/lxc.sh.rst @@ -0,0 +1,148 @@ + +.. _snap: https://snapcraft.io +.. _snapcraft LXD: https://snapcraft.io/lxd +.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/ +.. _LXC: https://linuxcontainers.org/lxc/introduction/ +.. _LXD: https://linuxcontainers.org/lxd/introduction/ +.. _`LXD@github`: https://github.com/lxc/lxd + +.. _archlinux: https://www.archlinux.org/ + +.. _lxc.sh: + +================ +``utils/lxc.sh`` +================ + +.. sidebar:: further reading + + - snap_, `snapcraft LXD`_ + - LXC_, LXD_ + - `LXC/LXD Image Server`_ + - `LXD@github`_ + +With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of +containers, what we call the: *lxc suite*. The *searx suite* +(:origin:`lxc-searx.env `) is loaded by default, every time +you start the ``lxc.sh`` script (*you do not need to care about*). + +Before you can start with containers, you need to install and initiate LXD_ +once:: + + $ snap install lxd + $ lxd init --auto + +To make use of the containers from the *searx suite*, you have to build the +:ref:`LXC suite containers ` initial. But be warned, **this might +take some time**:: + + $ sudo -H ./utils/lxc.sh build + +A cup of coffee later, your LXC suite is build up and you can run whatever task +you want / in a selected or even in all :ref:`LXC suite containers `. If you do not want to build all containers, **you can build just +one**:: + + $ sudo -H ./utils/lxc.sh build searx-ubu1804 + +*Good to know ...* + +Eeach container shares the root folder of the repository and the +command ``utils/lxc.sh cmd`` **handles relative path names transparent**, +compare output of:: + + $ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile + ... + +In the containers, you can run what ever you want, e.g. to start a bash use:: + + $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash + INFO: [searx-ubu1804] bash + root@searx-ubu1804:/share/searx# + +If there comes the time you want to **get rid off all** the containers and +**clean up local images** just type:: + + $ sudo -H ./utils/lxc.sh remove + $ sudo -H ./utils/lxc.sh remove images + + +Install suite +============= + +To install the complete :ref:`searx suite (includes searx, morty & filtron) +` into all LXC_ use:: + + $ sudo -H ./utils/lxc.sh install suite + +The command above installs a searx suite (see :ref:`installation scripts`). To +get the IP (URL) of the filtron service in the containers use ``show suite`` +command. To test instances from containers just open the URLs in your +WEB-Browser:: + + $ sudo ./utils/lxc.sh show suite | grep filtron + [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx + [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx + [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx + [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx + [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx + [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx + +To :ref:`install a nginx ` reverse proxy for filtron and +morty use (or alternatively use :ref:`apache `):: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install + + +Running commands +================ + +**Inside containers, you can use make or run scripts** from the +:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the +Makefile target ``test`` in the archlinux_ container:: + + sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost + sudo -H ./utils/lxc.sh cmd searx-archlinux make test + + +Setup searx buildhost +===================== + +You can **install the searx buildhost environment** into one or all containers. +The installation procedure to set up a :ref:`build host` takes its +time. Installation in all containers will take more time (time for another cup +of coffee).:: + + sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost + +To build (live) documentation inside a archlinux_ container:: + + sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live + ... + [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080 + +To get IP of the container and the port number *live docs* is listening:: + + $ sudo ./utils/lxc.sh show suite | grep docs-live + ... + [searx-archlinux] INFO: (eth0) docs-live: http://n.n.n.12:8080/ + + +.. _lxc.sh help: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory: + +.. program-output:: ../utils/lxc.sh --help + + +.. _lxc-searx.env: + +searx suite +=========== + +.. literalinclude:: ../../utils/lxc-searx.env + :language: bash diff --git a/docs/utils/morty.sh.rst b/docs/utils/morty.sh.rst new file mode 100644 index 00000000..3bdf9bdd --- /dev/null +++ b/docs/utils/morty.sh.rst @@ -0,0 +1,80 @@ + +.. _morty: https://github.com/asciimoo/morty +.. _morty's README: https://github.com/asciimoo/morty +.. _Go: https://golang.org/ + +.. _morty.sh: + +================== +``utils/morty.sh`` +================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` (:ref:`nginx ` & :ref:`apache + `) + - :ref:`searx morty` + +To simplify installation and maintenance of a morty_ instance you can use the +script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by +running the command: + +.. code:: bash + + sudo -H ./utils/morty.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_ +into this user account: + +#. Create a separated user account (``morty``). +#. Download and install Go_ binary in user's $HOME (``~morty``). +#. Install morty_ with the package management from Go_ (``go get -v -u + github.com/asciimoo/morty``) +#. Setup a systemd service unit :origin:`[ref] + ` + (``/lib/systemd/system/morty.service``). + +.. hint:: + + To add morty to your searx instance read chapter :ref:`searx morty`. + +Create user +=========== + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START create user + :end-before: END create user + + +Install go +========== + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START install go + :end-before: END install go + + +Install morty +============= + +Install morty software and systemd unit: + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START install morty + :end-before: END install morty + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START install systemd unit + :end-before: END install systemd unit + +.. _morty.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/morty.sh --help + diff --git a/docs/utils/searx.sh.rst b/docs/utils/searx.sh.rst new file mode 100644 index 00000000..dd4442f9 --- /dev/null +++ b/docs/utils/searx.sh.rst @@ -0,0 +1,39 @@ + +.. _searx.sh: + +================== +``utils/searx.sh`` +================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` + - :ref:`installation nginx` + - :ref:`installation apache` + +To simplify installation and maintenance of a searx instance you can use the +script :origin:`utils/searx.sh`. + +Install +======= + +In most cases you will install searx simply by running the command: + +.. code:: bash + + sudo -H ./utils/searx.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs searx +into this user account. The installation is described in chapter +:ref:`installation basic`. + +.. _intranet reverse proxy: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/searx.sh --help diff --git a/requirements-dev.txt b/requirements-dev.txt index eed7ec3d..5a1ca159 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,5 @@ selenium==3.141.0 linuxdoc @ git+http://github.com/return42/linuxdoc.git sphinx-jinja sphinx-tabs +sphinxcontrib-programoutput +twine diff --git a/searx/brand.py b/searx/brand.py index d563447f..91d2ab32 100644 --- a/searx/brand.py +++ b/searx/brand.py @@ -1,4 +1,5 @@ GIT_URL = 'https://github.com/asciimoo/searx' +GIT_BRANCH = 'master' ISSUE_URL = 'https://github.com/asciimoo/searx/issues' SEARX_URL = 'https://searx.me' DOCS_URL = 'https://asciimoo.github.io/searx' diff --git a/utils/brand.env b/utils/brand.env index 7fe1a391..5d5b3951 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,4 +1,5 @@ export GIT_URL='https://github.com/asciimoo/searx' +export GIT_BRANCH='master' export ISSUE_URL='https://github.com/asciimoo/searx/issues' export SEARX_URL='https://searx.me' export DOCS_URL='https://asciimoo.github.io/searx' diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 00000000..8986fb0e --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,561 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2119,SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" +PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" + +FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ + +FILTRON_ETC="/etc/filtron" +FILTRON_RULES="$FILTRON_ETC/rules.json" + +FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" +FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" +FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +APACHE_FILTRON_SITE="searx.conf" +NGINX_FILTRON_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <&1 | prefix_stdout + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 < "$GO_ENV" <> ~/.profile +EOF +} + +filtron_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] +} + +_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +install_filtron() { + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF +} + +inspect_service() { + + rst_title "service status & log" + + cat < ${PUBLIC_URL}" + info_msg "internal URL --> http://${FILTRON_LISTEN}" + fi + + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + + +enable_debug() { + info_msg "try to enable debug mode ..." + python </dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + +DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}" + +source_dot_config() { + if [[ ! -e "${DOT_CONFIG}" ]]; then + err_msg "configuration does not extsts at: ${DOT_CONFIG}" + return 42 + fi + # shellcheck disable=SC1090 + source "${DOT_CONFIG}" +} + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +required_commands() { + + # usage: required_commands [cmd1 ...] + + local exit_val=0 + while [ -n "$1" ]; do + + if ! command -v "$1" &>/dev/null; then + err_msg "missing command $1" + exit_val=42 + fi + shift + done + return $exit_val +} + +# colors +# ------ + +# shellcheck disable=SC2034 +set_terminal_colors() { + _colors=8 + _creset='\e[0m' # reset all attributes + + _Black='\e[0;30m' + _White='\e[1;37m' + _Red='\e[0;31m' + _Green='\e[0;32m' + _Yellow='\e[0;33m' + _Blue='\e[0;34m' + _Violet='\e[0;35m' + _Cyan='\e[0;36m' + + _BBlack='\e[1;30m' + _BWhite='\e[1;37m' + _BRed='\e[1;31m' + _BGreen='\e[1;32m' + _BYellow='\e[1;33m' + _BBlue='\e[1;34m' + _BPurple='\e[1;35m' + _BCyan='\e[1;36m' +} + +if [ ! -p /dev/stdout ]; then + set_terminal_colors +fi + +# reST +# ---- + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_title() { + # usage: rst_title [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${_BGreen}${1//?/=}${_creset}\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";; + chapter) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";; + section) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/-}${_creset}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +die() { + echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2; + exit "${1-1}" +} + +die_caller() { + echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2; + exit "${1-1}" +} + +err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } +warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } +info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } + +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while read -r -n1 -t 0.1; do : ; done + fi +} + +wait_key(){ + # usage: waitKEY [] + + clean_stdin + local _t=$1 + local msg="${MSG}" + [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}" + + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + printf "$msg" + # shellcheck disable=SC2086 + read -r -s -n1 $_t + echo + clean_stdin +} + +ask_yn() { + # usage: ask_yn [Ny|Yn] [] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + case "${FORCE_SELECTION:-${2}}" in + Y) return ${EXIT_YES} ;; + N) return ${EXIT_NO} ;; + Yn) + local exit_val=${EXIT_YES} + local choice="[${_BGreen}YES${_creset}/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[${_BGreen}NO${_creset}/yes]" + local default="No" + ;; + esac + echo + while true; do + clean_stdin + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -r -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + clean_stdin + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <>> print("hello") + # hello + + local _t="0"; + if [[ -n $1 ]] ; then _t="$1"; fi + + (while read -r line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: | prefix_stdout [prefix] + + local prefix="${_BYellow}-->|${_creset}" + + if [[ -n $1 ]] ; then prefix="$1"; fi + + # shellcheck disable=SC2162 + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download + + local exit_value=0 + + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if [[ ! $exit_value = 0 ]]; then + err_msg "failed to download: $1" + fi + fi +} + +backup_file() { + + # usage: backup_file /path/to/file.foo + + local stamp + stamp=$(date +"_%Y%m%d_%H%M%S") + info_msg "create backup: ${1}${stamp}" + cp -a "${1}" "${1}${stamp}" +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "${_BGreen}Menu::${_creset}" + for ((i=1; i<= $((max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + clean_stdin + printf "$1 [${_BGreen}$default${_creset}] " + + if (( 10 > max )); then + # shellcheck disable=SC2086 + read -r -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read -r $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + eval "$env_name"='${list[${REPLY}]}' + echo + clean_stdin +} + +install_template() { + + # usage: + # + # install_template [--no-eval] [--variant=] \ + # {file} [{owner} [{group} [{chmod}]]] + # + # E.g. the origin of variant 'raw' of /etc/updatedb.conf is:: + # + # ${TEMPLATES}/etc/updatedb.conf:raw + # + # To install variant 'raw' of /etc/updatedb.conf without evaluated + # replacements you can use:: + # + # install_template --variant=raw --no-eval \ + # /etc/updatedb.conf root root 644 + + local _reply="" + local do_eval=1 + local variant="" + local pos_args=("$0") + + for i in "$@"; do + case $i in + --no-eval) do_eval=0; shift ;; + --variant=*) variant=":${i#*=}"; shift ;; + *) pos_args+=("$i") ;; + esac + done + + local dst="${pos_args[1]}" + local template_origin="${TEMPLATES}${dst}${variant}" + local template_file="${TEMPLATES}${dst}" + + local owner="${pos_args[2]-$(id -un)}" + local group="${pos_args[3]-$(id -gn)}" + local chmod="${pos_args[4]-644}" + + info_msg "install (eval=$do_eval): ${dst}" + [[ -n $variant ]] && info_msg "variant --> ${variant}" + + if [[ ! -f "${template_origin}" ]] ; then + err_msg "${template_origin} does not exists" + err_msg "... can't install $dst" + wait_key 30 + return 42 + fi + + if [[ "$do_eval" == "1" ]]; then + template_file="${CACHE}${dst}${variant}" + info_msg "BUILD template ${template_file}" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")" + else + mkdir -p "$(dirname "${template_file}")" + fi + # shellcheck disable=SC2086 + eval "echo \"$(cat ${template_origin})\"" > "${template_file}" + if [[ -n ${SUDO_USER} ]]; then + chown "${SUDO_USER}:${SUDO_USER}" "${template_file}" + fi + else + template_file=$template_origin + fi + + mkdir -p "$(dirname "${dst}")" + + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "diffrent file ${dst} allready exists on this host" + + while true; do + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" \ + "interactiv shell" \ + "diff files" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + break + ;; + "leave file unchanged") + break + ;; + "interactiv shell") + echo -e "// edit ${_Red}${dst}${_creset} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${owner}" -i + $DIFF_CMD "${dst}" "${template_file}" + echo + echo -e "// ${_BBlack}did you edit file ...${_creset}" + echo -en "// ${_Red}${dst}${_creset}" + if ask_yn "//${_BBlack}... to your needs?${_creset}"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout + esac + done +} + + +service_is_available() { + + # usage: service_is_available + + [[ -z $1 ]] && die_caller 42 "missing argument " + local URL="$1" + http_code=$(curl -H 'Cache-Control: no-cache' \ + --silent -o /dev/null --head --write-out '%{http_code}' --insecure \ + "${URL}") + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "got $http_code from ${URL}" + fi + case "$http_code" in + 404|410|423) exit_val=$http_code;; + esac + return "$exit_val" +} + +# golang +# ------ + +go_is_available() { + + # usage: go_is_available $SERVICE_USER && echo "go is installed!" + + sudo -i -u "${1}" which go &>/dev/null +} + +install_go() { + + # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + + local _svcpr=" ${_Yellow}|${3}|${_creset} " + + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${1}" "${2}" + + tee_stderr 0.1 </dev/null && echo "ERROR - Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF +} + +# system accounts +# --------------- + +service_account_is_available() { + + # usage: service_account_is_available "$SERVICE_USER" && echo "OK" + + sudo -i -u "$1" echo \$HOME &>/dev/null +} + +drop_service_account() { + + # usage: drop_service_account "${SERVICE_USER}" + + rst_title "Drop ${1} HOME" section + if ask_yn "Do you really want to drop ${1} home folder?"; then + userdel -r -f "${1}" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "${1}") unchanged." + fi +} + +interactive_shell(){ + + # usage: interactive_shell "${SERVICE_USER}" + + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${1}" -i +} + + +# systemd +# ------- + +SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}" + +systemd_install_service() { + + # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + rst_title "Install System-D Unit ${1}" section + echo + install_template "${2}" root root 644 + wait_key + systemd_activate_service "${1}" +} + +systemd_remove_service() { + + # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then + return 42 + fi + systemd_deactivate_service "${1}" + rm "${2}" 2>&1 | prefix_stdout +} + +systemd_activate_service() { + + # usage: systemd_activate_service "${SERVICE_NAME}" + + rst_title "Activate ${1} (service)" section + echo + tee_stderr <&1 +systemctl enable ${1}.service +systemctl restart ${1}.service +EOF + tee_stderr <&1 +systemctl status --no-pager ${1}.service +EOF +} + +systemd_deactivate_service() { + + # usage: systemd_deactivate_service "${SERVICE_NAME}" + + rst_title "De-Activate ${1} (service)" section + echo + tee_stderr <&1 | prefix_stdout +systemctl stop ${1}.service +systemctl disable ${1}.service +EOF +} + +systemd_restart_service() { + + # usage: systemd_restart_service "${SERVICE_NAME}" + + rst_title "Restart ${1} (service)" section + echo + tee_stderr <&1 +systemctl restart ${1}.service +EOF + tee_stderr <&1 +systemctl status --no-pager ${1}.service +EOF +} + + +# nginx +# ----- + +nginx_distro_setup() { + # shellcheck disable=SC2034 + + NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf + + # Including *location* directives from a dedicated config-folder into the + # server directive is, what what fedora (already) does. + NGINX_APPS_ENABLED="/etc/nginx/default.d" + + # We add a apps-available folder and linking configurations into the + # NGINX_APPS_ENABLED folder. See also nginx_include_apps_enabled(). + NGINX_APPS_AVAILABLE="/etc/nginx/default.apps-available" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + NGINX_PACKAGES="nginx" + NGINX_DEFAULT_SERVER=/etc/nginx/sites-available/default + ;; + arch-*) + NGINX_PACKAGES="nginx-mainline" + ;; + fedora-*) + NGINX_PACKAGES="nginx" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: nginx not yet implemented" + ;; + esac +} +nginx_distro_setup + +install_nginx(){ + info_msg "installing nginx ..." + pkg_install "${NGINX_PACKAGES}" + case $DIST_ID-$DIST_VERS in + arch-*|fedora-*) + systemctl enable nginx + systemctl start nginx + ;; + esac +} + +nginx_is_installed() { + command -v nginx &>/dev/null +} + +nginx_reload() { + + info_msg "reload nginx .." + echo + if ! nginx -t; then + err_msg "testing nginx configuration failed" + return 42 + fi + systemctl restart nginx +} + +nginx_install_app() { + + # usage: nginx_install_app [