From c3e4753ce951f759844db13d98f9ad5b226f84b6 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 2 Mar 2020 19:00:19 +0100 Subject: [PATCH] docs: generic documentation from the installation scripts Signed-off-by: Markus Heiser --- Makefile | 12 +- docs/admin/index.rst | 4 + docs/admin/installation-docker.rst | 28 ++++ docs/admin/installation-searx.rst | 85 ++++++++++ docs/admin/installation-uwsgi.rst | 82 ++++++++++ docs/admin/installation.rst | 247 +++++------------------------ docs/admin/update-searx.rst | 23 +++ docs/utils/index.rst | 9 -- utils/filtron.sh | 4 + utils/lib.sh | 8 +- utils/lxc.sh | 5 + utils/morty.sh | 4 + utils/searx.sh | 214 ++++++++++++++++++++++--- 13 files changed, 484 insertions(+), 241 deletions(-) create mode 100644 docs/admin/installation-docker.rst create mode 100644 docs/admin/installation-searx.rst create mode 100644 docs/admin/installation-uwsgi.rst create mode 100644 docs/admin/update-searx.rst diff --git a/Makefile b/Makefile index 1d26ef18..fadf80cd 100644 --- a/Makefile +++ b/Makefile @@ -60,13 +60,21 @@ run: pyenvinstall # ---- PHONY += docs -docs: pyenvinstall sphinx-doc + +docs: pyenvinstall sphinx-doc prebuild-includes $(call cmd,sphinx,html,docs,docs) PHONY += docs-live -docs-live: pyenvinstall sphinx-live +docs-live: pyenvinstall sphinx-live prebuild-includes $(call cmd,sphinx_autobuild,html,docs,docs) +prebuild-includes: + @mkdir -p $(DOCS_BUILD)/includes + @./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst + @./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst + @./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst + + $(GH_PAGES):: @echo "doc available at --> $(DOCS_URL)" diff --git a/docs/admin/index.rst b/docs/admin/index.rst index 4190bf24..c708c4ff 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -7,8 +7,12 @@ Administrator documentation :caption: Contents installation + installation-searx + installation-uwsgi installation-nginx installation-apache + installation-docker + update-searx settings api architecture diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst new file mode 100644 index 00000000..340e6631 --- /dev/null +++ b/docs/admin/installation-docker.rst @@ -0,0 +1,28 @@ +.. _installation docker: + +=================== +Docker installation +=================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Make sure you have installed Docker. For instance, you can deploy searx like this: + +.. code:: sh + + docker pull wonderfall/searx + docker run -d --name searx -p $PORT:8888 wonderfall/searx + +Go to ``http://localhost:$PORT``. + +See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also +possible to build searx from the embedded Dockerfile. + +.. code:: sh + + git clone https://github.com/asciimoo/searx.git + cd searx + docker build -t whatever/searx . diff --git a/docs/admin/installation-searx.rst b/docs/admin/installation-searx.rst new file mode 100644 index 00000000..11018bb3 --- /dev/null +++ b/docs/admin/installation-searx.rst @@ -0,0 +1,85 @@ +.. _installation basic: + +========================= +Step by step installation +========================= + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Step by step installation with virtualenv. For Ubuntu, be sure to have enable +universe repository. + +Install packages +================ + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START distro-packages + :end-before: END distro-packages + +.. hint:: + + This installs also the packages needed by :ref:`searx uwsgi` + +Create user +=========== + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START create user + :end-before: END create user + +Start a interactive shell from new created user and clone searx: + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START clone searx + :end-before: END clone searx + +In the same shell create *virtualenv*: + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START create virtualenv + :end-before: END create virtualenv + +To install searx's dependencies, exit the searx *bash* session you opened above +and restart a new. Before install, first check if your *virualenv* was sourced +from the login (*~/.profile*): + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START manage.sh update_packages + :end-before: END manage.sh update_packages + +.. tip:: + + Open a second terminal for the configuration tasks and left the ``(searx)$`` + terminal open for the tasks below. + + +Configuration +============== + +Create a copy of the :origin:`searx/settings.yml` configuration file in system's +*/etc* folder. Configure like shown below -- replace ``searx@\$(uname -n)`` with +a name of your choice -- *and/or* edit ``/etc/searx/settings.yml`` if necessary. + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START searx config + :end-before: END searx config + + +Check +===== + +To check your searx setup, optional enable debugging and start the *webapp*. +Searx looks at the exported environment ``$SEARX_SETTINGS_PATH`` for a +configuration file. + +.. include:: ../../build/docs/includes/searx.rst + :start-after: START check searx installation + :end-before: END check searx installation + +If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the +debug option in ``settings.yml``. You can now exit searx user bash (enter exit +command twice). At this point searx is not demonized; uwsgi allows this. + diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst new file mode 100644 index 00000000..0ee4800a --- /dev/null +++ b/docs/admin/installation-uwsgi.rst @@ -0,0 +1,82 @@ +.. _searx uwsgi: + +===== +uwsgi +===== + +Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this +content: + +.. code:: ini + + [uwsgi] + + # uWSGI core + # ---------- + # + # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + + # Who will run the code + uid = searx + gid = searx + + # chdir to specified directory before apps loading + chdir = /usr/local/searx/searx-src/searx + + # disable logging for privacy + disable-logging = true + + # The right granted on the created socket + chmod-socket = 666 + + # Plugin to use and interpretor config + single-interpreter = true + + # enable master process + master = true + + # load apps in each worker instead of the master + lazy-apps = true + + # load uWSGI plugins + plugin = python3,http + + # By default the Python plugin does not initialize the GIL. This means your + # app-generated threads will not run. If you need threads, remember to enable + # them with enable-threads. Running uWSGI in multithreading mode (with the + # threads options) will automatically enable threading support. This *strange* + # default behaviour is for performance reasons. + enable-threads = true + + # plugin: python + # -------------- + # + # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + + # load a WSGI module + module = searx.webapp + + # set PYTHONHOME/virtualenv + virtualenv = /usr/local/searx/searx-pyenv + + # add directory (or glob) to pythonpath + pythonpath = /usr/local/searx/searx-src + + + # plugin http + # ----------- + # + # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http + + # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + http = 127.0.0.1:8888 + +Activate the uwsgi application and restart: + +.. code:: sh + + cd /etc/uwsgi/apps-enabled + ln -s ../apps-available/searx.ini + /etc/init.d/uwsgi restart + + diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index a4c7eb8b..330a51fd 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -4,234 +4,63 @@ Installation ============ -.. sidebar:: Searx server setup +*You're spoilt for choice*, choose your preferred method of installation. - - :ref:`installation nginx` - - :ref:`installation apache` +- :ref:`installation docker` +- `Installation scripts`_ +- :ref:`installation basic` - If you do not have any special preferences, it is recommend to use - :ref:`searx.sh`. +The :ref:`installation basic` is good enough for intranet usage and it is a +excellent illustration of *how a searx instance is build up*. If you place your +instance public to the internet you should really consider to install a +:ref:`filtron reverse proxy ` and for privacy a :ref:`result proxy +` is mandatory. -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry +Therefore, if you do not have any special preferences, its recommend to use the +:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling +box ` as described below. -.. _installation basic: -Basic installation -================== +Installation scripts +==================== -Step by step installation with virtualenv. For Ubuntu, be sure to have enable -universe repository. +The following will install a setup as shown in :ref:`architecture`. First you +need to get a clone. The clone is only needed for the installation procedure +and some maintenance tasks (alternatively you can create your own fork). -Install packages: +.. code:: bash -.. tabs:: + $ cd ~/Download + $ git clone https://github.com/asciimoo/searx searx + $ cd searx - .. group-tab:: Ubuntu / debian +.. hint:: - .. code-block:: sh + The *tooling box* is not yet merged into `asciimoo/searx master + `_. As long as PR is not merged, you need + to merge the PR into your local clone (see below). The discussion takes + place in :pull:`1803`. To merge the :pull:`1803` in your local branch use: - $ sudo -H apt-get install -m \ - git build-essential - libxslt-dev python3-dev python3-babel python3-venv \ - zlib1g-dev libffi-dev libssl-dev + .. code:: bash -Install searx: + $ git pull origin refs/pull/1803/head -.. code:: sh +**Install** :ref:`searx service ` - sudo -H useradd searx --system --disabled-password -d /usr/local/searx - sudo -H usermod -a -G shadow searx - cd /usr/local/searx - sudo -H git clone https://github.com/asciimoo/searx.git searx-src - sudo -H chown searx:searx -R /usr/local/searx +This installs searx as described in :ref:`installation basic`. -Install virtualenv: +.. code:: bash -.. code:: sh + $ sudo -H ./utils/searx.sh install all - sudo -H -u searx -i - (searx)$ python3 -m venv searx-pyenv - (searx)$ echo 'source ~/searx-pyenv/bin/activate' > ~/.profile +**Install** :ref:`filtron reverse proxy ` -Exit the searx bash and restart a new to install the searx dependencies: +.. code:: bash -.. code:: sh + $ sudo -H ./utils/filtron.sh install all - sudo -H -u searx -i - (searx)$ cd searx-src - (searx)$ ./manage.sh update_packages +**Install** :ref:`result proxy ` -Configuration -============== +.. code:: bash -.. code:: sh - - sudo -H -u searx -i - (searx)$ cd searx-src - (searx)$ sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml - -Edit searx/settings.yml if necessary. - -Check -===== - -Start searx: - -.. code:: sh - - sudo -H -u searx -i - (searx)$ cd searx-src - (searx)$ python3 searx/webapp.py - -Go to http://localhost:8888 - -If everything works fine, disable the debug option in settings.yml: - -.. code:: sh - - sed -i -e "s/debug : True/debug : False/g" searx/settings.yml - -At this point searx is not demonized ; uwsgi allows this. You can exit the -virtualenv and the searx user bash (enter exit command twice). - -uwsgi -===== - -Install packages: - -.. tabs:: - - .. group-tab:: Ubuntu / debian - - .. code-block:: bash - - sudo -H apt-get install uwsgi uwsgi-plugin-python3 - -Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this -content: - -.. code:: ini - - [uwsgi] - - # uWSGI core - # ---------- - # - # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core - - # Who will run the code - uid = searx - gid = searx - - # chdir to specified directory before apps loading - chdir = /usr/local/searx/searx-src/searx - - # disable logging for privacy - disable-logging = true - - # The right granted on the created socket - chmod-socket = 666 - - # Plugin to use and interpretor config - single-interpreter = true - - # enable master process - master = true - - # load apps in each worker instead of the master - lazy-apps = true - - # load uWSGI plugins - plugin = python3,http - - # By default the Python plugin does not initialize the GIL. This means your - # app-generated threads will not run. If you need threads, remember to enable - # them with enable-threads. Running uWSGI in multithreading mode (with the - # threads options) will automatically enable threading support. This *strange* - # default behaviour is for performance reasons. - enable-threads = true - - # plugin: python - # -------------- - # - # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python - - # load a WSGI module - module = searx.webapp - - # set PYTHONHOME/virtualenv - virtualenv = /usr/local/searx/searx-pyenv - - # add directory (or glob) to pythonpath - pythonpath = /usr/local/searx/searx-src - - - # plugin http - # ----------- - # - # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http - - # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html - http = 127.0.0.1:8888 - -Activate the uwsgi application and restart: - -.. code:: sh - - cd /etc/uwsgi/apps-enabled - ln -s ../apps-available/searx.ini - /etc/init.d/uwsgi restart - - -How to update -============= - -.. code:: sh - - sudo -H -u searx -i - (searx)$ git stash - (searx)$ git pull origin master - (searx)$ git stash apply - (searx)$ ./manage.sh update_packages - -Restart uwsgi: - -.. tabs:: - - .. group-tab:: Ubuntu / debian - - .. code:: sh - - sudo -H systemctl restart uwsgi - -Docker -====== - -Make sure you have installed Docker. For instance, you can deploy searx like this: - -.. code:: sh - - docker pull wonderfall/searx - docker run -d --name searx -p $PORT:8888 wonderfall/searx - -Go to ``http://localhost:$PORT``. - -See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also -possible to build searx from the embedded Dockerfile. - -.. code:: sh - - git clone https://github.com/asciimoo/searx.git - cd searx - docker build -t whatever/searx . - -References -========== - -* https://about.okhin.fr/posts/Searx/ with some additions - -* How to: `Setup searx in a couple of hours with a free SSL certificate - `__ + $ sudo -H ./utils/morty.sh install all diff --git a/docs/admin/update-searx.rst b/docs/admin/update-searx.rst new file mode 100644 index 00000000..c74c9568 --- /dev/null +++ b/docs/admin/update-searx.rst @@ -0,0 +1,23 @@ +.. _update searx: + +============= +How to update +============= + +.. code:: sh + + sudo -H -u searx -i + (searx)$ git stash + (searx)$ git pull origin master + (searx)$ git stash apply + (searx)$ ./manage.sh update_packages + +Restart uwsgi: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart uwsgi diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 33891471..1c77cbfc 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -1,4 +1,3 @@ - .. _searx_utils: .. _toolboxing: @@ -9,14 +8,6 @@ Tooling box ``utils/*`` In the folder :origin:`utils/` we maintain some tools useful for admins and developers. -.. sidebar:: Work needed! - - Our scripts to maintain services do most support only systemd init process - used by debian, ubuntu and many other dists. In general our scripts are only - partially usable on debian systems. We are working on this limitation, if - you have any contribution, please send us your :pull:`PR <../pulls>`, see - :ref:`how to contribute`. - .. toctree:: :maxdepth: 2 :caption: Contents diff --git a/utils/filtron.sh b/utils/filtron.sh index c53acc60..89971d61 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -180,6 +180,10 @@ main() { debug-off) echo; disable_debug ;; *) usage "$_usage"; exit 42;; esac ;; + doc) + echo + echo ".. generic utils/filtron.sh documentation" + ;; *) usage "unknown or missing command $1"; exit 42;; esac diff --git a/utils/lib.sh b/utils/lib.sh index 47e9e803..6de66bca 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -933,7 +933,9 @@ pkg_install() { # usage: TITEL='install foobar' pkg_install foopkg barpkg rst_title "${TITLE:-installation of packages}" section - echo -en "\npackage(s)::\n\n $*\n" | $FMT + echo -e "\npackage(s)::\n" + # shellcheck disable=SC2068 + echo " " $@ | $FMT if ! ask_yn "Should packages be installed?" Yn 30; then return 42 @@ -959,7 +961,9 @@ pkg_remove() { # usage: TITEL='remove foobar' pkg_remove foopkg barpkg rst_title "${TITLE:-remove packages}" section - echo -en "\npackage(s)::\n\n $*\n" | $FMT + echo -e "\npackage(s)::\n" + # shellcheck disable=SC2068 + echo " " $@ | $FMT if ! ask_yn "Should packages be removed (purge)?" Yn 30; then return 42 diff --git a/utils/lxc.sh b/utils/lxc.sh index f950cdb1..674eab4a 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -209,6 +209,11 @@ main() { searx-suite) searx_suite_install ;; esac ;; + doc) + echo + echo ".. generic utils/lxc.sh documentation" + ;; + *) usage "unknown or missing command $1"; exit 42;; esac diff --git a/utils/morty.sh b/utils/morty.sh index c3531f73..143e7e0c 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -183,6 +183,10 @@ main() { debug-off) disable_debug ;; *) usage "$_usage"; exit 42;; esac ;; + doc) + echo + echo ".. generic utils/morty.sh documentation" + ;; *) usage "ERROR: unknown or missing command $1"; exit 42;; esac diff --git a/utils/searx.sh b/utils/searx.sh index 1089f30d..de903bc0 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -36,25 +36,28 @@ SEARX_UWSGI_APP="searx.ini" # shellcheck disable=SC2034 SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" +# apt packages +SEARX_PACKAGES_debian="\ +python3-dev python3-babel python3-venv +uwsgi uwsgi-plugin-python3 +git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev" + +# pacman packages +SEARX_PACKAGES_arch="\ +python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python +git base-devel libxml2" + +# dnf packages +SEARX_PACKAGES_fedora="\ +python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python3 +git @development-tools libxml2" + case $DIST_ID in - ubuntu|debian) # apt packages - SEARX_PACKAGES="\ - python3-dev python3-babel python3-venv \ - uwsgi uwsgi-plugin-python3 \ - git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev " - ;; - arch) # pacman packages - SEARX_PACKAGES="\ - python python-pip python-lxml python-babel \ - uwsgi uwsgi-plugin-python \ - git base-devel libxml2 " - ;; - fedora) # dnf packages - SEARX_PACKAGES="\ - python python-pip python-lxml python-babel \ - uwsgi uwsgi-plugin-python3 \ - git @development-tools libxml2 " - ;; + ubuntu|debian) SEARX_PACKAGES="${SEARX_PACKAGES_debian}" ;; + arch) SEARX_PACKAGES="${SEARX_PACKAGES_arch}" ;; + fedora) SEARX_PACKAGES="${SEARX_PACKAGES_fedora}" ;; esac # Apache Settings @@ -206,6 +209,7 @@ main() { remove) remove_apache_site ;; *) usage "$_usage"; exit 42;; esac ;; + doc) rst-doc;; *) usage "unknown or missing command $1"; exit 42;; esac @@ -443,7 +447,7 @@ test_local_searx() { tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}" cd ${SEARX_SRC} -timeout 10 python3 searx/webapp.py & +timeout 10 python searx/webapp.py & sleep 3 curl --location --verbose --head --insecure $SEARX_INTERNAL_URL EOF @@ -631,6 +635,178 @@ This removes apache site ${APACHE_SEARX_SITE}." apache_remove_site "${APACHE_SEARX_SITE}" } +rst-doc() { + local debian="${SEARX_PACKAGES_debian}" + local arch="${SEARX_PACKAGES_arch}" + local fedora="${SEARX_PACKAGES_fedora}" + debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + + cat <> "$SERVICE_HOME/.profile" + +.. END create virtualenv + +.. START manage.sh update_packages + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H -u ${SERVICE_USER} -i + + (${SERVICE_USER})$ command -v python && python --version + $SEARX_PYENV/bin/python + Python 3.8.1 + + (${SERVICE_USER})$ cd "$SEARX_SRC" + (${SERVICE_USER})$ ./manage.sh update_packages + +.. END manage.sh update_packages + +.. START searx config + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H cp "$SEARX_SRC/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + $ sudo -H sed -i -e "s/ultrasecretkey/\$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" + $ sudo -H sed -i -e "s/{instance_name}/searx@\$(uname -n)/g" "$SEARX_SETTINGS_PATH" + +.. END searx config + +.. START check searx installation + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + # enable debug .. + $ sudo -H sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" + + # start webapp + $ sudo -H -u ${SERVICE_USER} -i + (${SERVICE_USER})$ cd ${SEARX_SRC} + (${SERVICE_USER})$ export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}" + (${SERVICE_USER})$ python searx/webapp.py + + # disable debug + $ sudo -H sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" + +Open WEB browser and visit http://$SEARX_INTERNAL_URL . If you are inside a +container or in a script, test with curl: + +.. tabs:: + + .. group-tab:: WEB browser + + .. code-block:: sh + + $ xgd-open http://$SEARX_INTERNAL_URL + + .. group-tab:: curl + + .. code-block:: none + + $ curl --location --verbose --head --insecure $SEARX_INTERNAL_URL + + * Trying 127.0.0.1:8888... + * TCP_NODELAY set + * Connected to 127.0.0.1 (127.0.0.1) port 8888 (#0) + > HEAD / HTTP/1.1 + > Host: 127.0.0.1:8888 + > User-Agent: curl/7.68.0 + > Accept: */* + > + * Mark bundle as not supporting multiuse + * HTTP 1.0, assume close after body + < HTTP/1.0 200 OK + HTTP/1.0 200 OK + ... + +.. END check searx installation + +EOF + +} + + # ---------------------------------------------------------------------------- main "$@" # ----------------------------------------------------------------------------