mirror of
git://git.savannah.gnu.org/guix/maintenance.git
synced 2023-12-14 03:33:04 +01:00
e672670e9a
This has existed for a while, but the config has been just on the machine. Since I'm looking at trying to sort out the hosting for this going forward, it seems sensible to bring the configuration in to maintenance.git, even though that's very overdue. * hydra/data-guix-gnu-org.scm: New file.
445 lines
16 KiB
Scheme
445 lines
16 KiB
Scheme
;; Configuration for the machine hosting data.guix.gnu.org
|
|
;;
|
|
;; Currently this is deployed on a Hetzner VM hosted in Nuremberg It
|
|
;; has 8 cores, 32GB of RAM and 240GB root storage with 600GB of
|
|
;; attached storage
|
|
;;
|
|
;; Copyright © 2020, 2021, 2022 Christopher Baines <mail@cbaines.net>
|
|
;; Released under the GNU GPLv3 or any later version.
|
|
|
|
(use-modules (srfi srfi-1)
|
|
(gnu)
|
|
(guix packages)
|
|
(guix build-system trivial)
|
|
(guix git-download)
|
|
(guix gexp))
|
|
(use-service-modules admin databases getmail guix networking mcron ssh
|
|
virtualization web mail monitoring certbot)
|
|
(use-package-modules certs screen ssh linux web bash version-control
|
|
haskell-apps python-web databases guile-xyz)
|
|
|
|
(define backup-script
|
|
(package
|
|
(name "data-guix-gnu-org-backup-script")
|
|
(version "0")
|
|
(source
|
|
(plain-file
|
|
"data-guix-gnu-org-backup"
|
|
"#!/bin/sh
|
|
|
|
set -eux
|
|
|
|
export GUIX_DATA_SERVICE_BACKUP_DIRECTORY=/var/lib/guix-data-service/dumps
|
|
export TMPDIR=/mnt/guix-data-service-temp-data
|
|
|
|
export GIT_SSH_COMMAND=\"ssh -F /etc/guix-data-service-backup-ssh-config/config\"
|
|
|
|
S3_ENDPOINT_URL=\"https://s3.eu-central-1.wasabisys.com\"
|
|
DATE=\"$(date \"+%Y-%m-%d\")\"
|
|
|
|
source /etc/wasabi-env-vars
|
|
|
|
cd \"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY\"
|
|
|
|
##
|
|
## Delete old backups
|
|
##
|
|
|
|
olderThan=`date -d\"-3 months\" +%s`
|
|
|
|
find \"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY/\" -maxdepth 1 -not -path '*/\\.*' -printf \"%f\\n\" | grep -v dumps | sort | while read -r line;
|
|
do
|
|
fileName=\"$line\"
|
|
createDate=`date -d\"$fileName\" +%s`
|
|
if [[ $createDate -lt $olderThan ]]
|
|
then
|
|
echo \"Deleting $fileName\"
|
|
if [[ $fileName != \"\" ]]
|
|
then
|
|
rm -r \"${GUIX_DATA_SERVICE_BACKUP_DIRECTORY}/${fileName}\"
|
|
git reset # Just to be careful not to commit anything else
|
|
git add \"${GUIX_DATA_SERVICE_BACKUP_DIRECTORY}/${fileName}\"
|
|
git commit -m \"Delete ${fileName}\"
|
|
aws s3 --endpoint-url=https://s3.eu-central-1.wasabisys.com rm s3://data-guix-gnu-org-database-dumps/ --recursive --exclude \"*\" --include \"${fileName}*\"
|
|
fi
|
|
fi
|
|
done;
|
|
|
|
##
|
|
## Main backup
|
|
##
|
|
|
|
guix-data-service-backup-database
|
|
|
|
# Add files to the annex
|
|
|
|
git annex add \"$DATE\"
|
|
git commit -m \"Add data for $DATE\"
|
|
|
|
git push
|
|
|
|
# Copy files manually to S3, as git annex export doesn't work reliably
|
|
|
|
aws s3 \"--endpoint-url=$S3_ENDPOINT_URL\" \\
|
|
cp \"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY/$DATE/guix_data_service_full.dump\" \\
|
|
\"s3://data-guix-gnu-org-database-dumps/$DATE/\"
|
|
|
|
# Run git annex export
|
|
|
|
git annex export master --to=wasabi-export || true
|
|
|
|
cleanup() {
|
|
echo \"cleaning up\"
|
|
git annex drop --force \"$DATE\"
|
|
}
|
|
|
|
trap \"cleanup\" ERR
|
|
|
|
##
|
|
## Small backup
|
|
##
|
|
|
|
#guix-data-service-create-small-backup \"$DATE\"
|
|
|
|
# Add files to the annex
|
|
|
|
#git annex add \"$DATE\"
|
|
#git commit -m \"Add small backup for $DATE\"
|
|
|
|
#git push
|
|
|
|
# Copy files manually to S3, as git annex export doesn't work reliably
|
|
|
|
#S3_ENDPOINT_URL=\"https://s3.eu-central-1.wasabisys.com\"
|
|
|
|
#aws s3 \"--endpoint-url=$S3_ENDPOINT_URL\" \\
|
|
cp \"$GUIX_DATA_SERVICE_BACKUP_DIRECTORY/$DATE/guix_data_service_small.dump\" \\
|
|
\"s3://data-guix-gnu-org-database-dumps/$DATE/\"
|
|
|
|
# Run git annex export
|
|
|
|
git annex export master --to=wasabi-export || true
|
|
|
|
##
|
|
## Cleanup
|
|
##
|
|
|
|
cleanup
|
|
"))
|
|
|
|
(build-system trivial-build-system)
|
|
(arguments
|
|
`(#:modules ((guix build utils)
|
|
(srfi srfi-26))
|
|
#:builder
|
|
(begin
|
|
(use-modules (guix build utils)
|
|
(srfi srfi-26))
|
|
(let* ((source (assoc-ref %build-inputs "source"))
|
|
(out (assoc-ref %outputs "out"))
|
|
(bin (string-append out "/bin"))
|
|
(script (string-append bin "/data-guix-gnu-org-backup")))
|
|
|
|
(setenv "PATH"
|
|
(string-append (assoc-ref %build-inputs "bash")
|
|
"/bin"))
|
|
|
|
(mkdir-p bin)
|
|
(copy-file source script)
|
|
(chmod script #o555)
|
|
(wrap-program
|
|
script
|
|
`("PATH" ":" prefix
|
|
,(map (lambda (dir)
|
|
(string-append dir "/bin"))
|
|
(map (lambda (input)
|
|
(assoc-ref %build-inputs input))
|
|
'("coreutils"
|
|
"git"
|
|
"git-annex"
|
|
"awscli"
|
|
"guix-data-service"))))))
|
|
#t)))
|
|
(inputs
|
|
`(("bash" ,bash)
|
|
("coreutils" ,coreutils)
|
|
("git" ,git)
|
|
("git-annex" ,git-annex)
|
|
("awscli" ,awscli)
|
|
("guix-data-service" ,guix-data-service)))
|
|
(home-page "")
|
|
(synopsis "")
|
|
(description "")
|
|
(license #f)))
|
|
|
|
(define %nginx-configuration
|
|
(nginx-configuration
|
|
(server-names-hash-bucket-size 64)
|
|
(extra-content
|
|
"
|
|
proxy_cache_path /var/cache/nginx/guix-data-service
|
|
levels=2
|
|
inactive=2d
|
|
keys_zone=guix-data-service:4m # ~32K keys
|
|
max_size=1g
|
|
use_temp_path=off;
|
|
|
|
limit_req_zone $binary_remote_addr zone=webpagezone:1m rate=1r/s;
|
|
")
|
|
(upstream-blocks
|
|
(list
|
|
(nginx-upstream-configuration
|
|
(name "guix-data-service-proxy")
|
|
(servers '("localhost:8765")))
|
|
(nginx-upstream-configuration
|
|
(name "wasabi-proxy")
|
|
(servers '("s3.eu-central-1.wasabisys.com:443")))))
|
|
(server-blocks
|
|
(list
|
|
(nginx-server-configuration
|
|
(listen '("80"))
|
|
(raw-content
|
|
(list "return 404;")))
|
|
(nginx-server-configuration
|
|
(server-name '("data.guix.gnu.org"))
|
|
(listen '("80" "443 ssl"))
|
|
(ssl-certificate
|
|
"/etc/letsencrypt/live/data.guix.gnu.org/fullchain.pem")
|
|
(ssl-certificate-key
|
|
"/etc/letsencrypt/live/data.guix.gnu.org/privkey.pem")
|
|
(locations
|
|
(list
|
|
(nginx-location-configuration
|
|
(uri "~ /(builds|statistics|revision|package|gnu)/")
|
|
(body '("proxy_pass http://guix-data-service-proxy;"
|
|
"proxy_http_version 1.1;"
|
|
"proxy_set_header Host $host;"
|
|
"proxy_set_header X-Forwarded-For $remote_addr;"
|
|
|
|
"proxy_cache guix-data-service;"
|
|
"proxy_cache_revalidate on;"
|
|
"proxy_cache_min_uses 3;"
|
|
"proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;"
|
|
"proxy_cache_background_update on;"
|
|
"proxy_cache_lock on;"
|
|
"add_header X-Cache-Status $upstream_cache_status;"
|
|
|
|
"keepalive_requests 20000;"
|
|
|
|
"limit_req zone=webpagezone burst=20 nodelay;"
|
|
"limit_req_status 429;"
|
|
|
|
"gzip on;"
|
|
"gzip_types text/html text/css application/json;"
|
|
"gzip_proxied any;")))
|
|
(nginx-location-configuration
|
|
(uri "/")
|
|
(body '("proxy_pass http://guix-data-service-proxy;"
|
|
"proxy_http_version 1.1;"
|
|
"proxy_set_header Host $host;"
|
|
"proxy_set_header X-Forwarded-For $remote_addr;"
|
|
|
|
"proxy_cache guix-data-service;"
|
|
"proxy_cache_revalidate on;"
|
|
"proxy_cache_min_uses 3;"
|
|
"proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;"
|
|
"proxy_cache_background_update on;"
|
|
"proxy_cache_lock on;"
|
|
"add_header X-Cache-Status $upstream_cache_status;"
|
|
|
|
"keepalive_requests 20000;"
|
|
|
|
"gzip on;"
|
|
"gzip_types text/html text/css application/json;"
|
|
"gzip_proxied any;")))
|
|
(nginx-location-configuration
|
|
(uri "~ ^/dumps/(?!latest)(.+)")
|
|
(body '("rewrite /dumps/(.+) /data-guix-gnu-org-database-dumps/$1 break;"
|
|
"proxy_pass https://wasabi-proxy;"
|
|
"proxy_set_header Host s3.eu-central-1.wasabisys.com;")))
|
|
(nginx-location-configuration
|
|
(uri "^~ /.well-known/acme-challenge/")
|
|
(body '("root /srv/http/data.guix.gnu.org;")))
|
|
(nginx-location-configuration
|
|
(uri "= /.well-known/acme-challenge/")
|
|
(body '("return 404;"))))))))))
|
|
|
|
(operating-system
|
|
(host-name "guix-hetzner-1")
|
|
(timezone "Europe/Berlin")
|
|
(locale "en_US.utf8")
|
|
|
|
(initrd-modules (cons "virtio_scsi" %base-initrd-modules))
|
|
|
|
;; Boot in "legacy" BIOS mode, assuming /dev/sdX is the
|
|
;; target hard disk, and "my-root" is the label of the target
|
|
;; root file system.
|
|
(bootloader (bootloader-configuration
|
|
(bootloader grub-bootloader)
|
|
(targets '("/dev/sda"))))
|
|
(file-systems (cons* (file-system
|
|
(device (file-system-label "root"))
|
|
(mount-point "/")
|
|
(type "ext4"))
|
|
;; sudo mount -o discard,defaults /dev/disk/by-id/scsi-0HC_Volume_4469096 /mnt/guix-data-service-temp-data
|
|
(file-system
|
|
(device "/dev/disk/by-id/scsi-0HC_Volume_4469096")
|
|
(mount-point "/mnt/guix-data-service-temp-data")
|
|
(type "ext4")
|
|
(options "discard"))
|
|
;; sudo mount -o discard,defaults /dev/disk/by-id/scsi-0HC_Volume_5374344 /mnt/guix-data-service-postgresql-additional-data
|
|
(file-system
|
|
(device "/dev/disk/by-id/scsi-0HC_Volume_5374344")
|
|
(mount-point "/mnt/guix-data-service-postgresql-additional-data")
|
|
(type "ext4")
|
|
(options "discard"))
|
|
%base-file-systems))
|
|
|
|
(swap-devices
|
|
(list (swap-space
|
|
(target "/swapfile"))))
|
|
|
|
(users (cons* (user-account
|
|
(name "chris")
|
|
(group "users")
|
|
(supplementary-groups '("wheel"
|
|
"audio" "video")))
|
|
%base-user-accounts))
|
|
|
|
(packages (cons* screen
|
|
nss-certs ; Needed for the Guix Data Service
|
|
%base-packages))
|
|
|
|
(services
|
|
(append (list
|
|
(service dhcp-client-service-type)
|
|
(service ntp-service-type)
|
|
(service openssh-service-type
|
|
(openssh-configuration
|
|
(password-authentication? #f)))
|
|
(service prometheus-node-exporter-service-type)
|
|
(service opensmtpd-service-type
|
|
(opensmtpd-configuration
|
|
(config-file
|
|
(plain-file "smtpd.conf" (simple-format #f "
|
|
table cred file:/etc/smtpd-secrets
|
|
listen on lo
|
|
action \"relay\" relay host \"smtp+tls://guixdataservice@smtp.cbaines.net:587\" auth <cred>
|
|
match for any action \"relay\"")))))
|
|
|
|
(service postgresql-service-type
|
|
(postgresql-configuration
|
|
(postgresql postgresql-13)
|
|
(config-file
|
|
(postgresql-config-file
|
|
(log-destination "stderr")
|
|
(hba-file
|
|
(plain-file "pg_hba.conf"
|
|
"
|
|
local all all trust
|
|
host all all 127.0.0.1/32 md5
|
|
host all all ::1/128 md5
|
|
"))
|
|
(extra-config
|
|
'(("session_preload_libraries" "auto_explain")
|
|
("auto_explain.log_min_duration" "500ms")
|
|
("random_page_cost" 1.0)
|
|
("work_mem" "500MB")
|
|
;; ("shared_buffers" "1G")
|
|
("max_worker_processes" 4)
|
|
("max_parallel_workers_per_gather" 2)
|
|
("effective_io_concurrency" 200)
|
|
("autovacuum_vacuum_threshold" 0)
|
|
("autovacuum_vacuum_cost_limit" 2000)
|
|
("logging_collector" #t)
|
|
("log_directory" "/var/log/postgresql")))))))
|
|
(service mcron-service-type
|
|
(mcron-configuration
|
|
(jobs (list
|
|
#~(job "30 2 * * *"
|
|
"guix gc -F 50G")
|
|
#~(job "0 1 */14 * *"
|
|
"guix gc")
|
|
#~(job "45 23 * * */2"
|
|
"data-guix-gnu-org-backup"
|
|
#:user "guix-data-service")))))
|
|
|
|
(simple-service 'guix-data-service-backup-script
|
|
profile-service-type
|
|
(list backup-script))
|
|
|
|
(service qemu-binfmt-service-type
|
|
(qemu-binfmt-configuration
|
|
(platforms
|
|
(lookup-qemu-platforms
|
|
"arm"
|
|
"aarch64"
|
|
"ppc64le"
|
|
;;"riscv64"
|
|
;; I'm not sure which mips thing is relevant
|
|
;; "mips" "mipsel" "mipsn32" "mipsn32el" "mips64" "mips64el"
|
|
))))
|
|
|
|
(service guix-data-service-type
|
|
(guix-data-service-configuration
|
|
(extra-options
|
|
'("--thread-pool-threads=32"))
|
|
(extra-process-jobs-options
|
|
'("--max-processes=2"
|
|
"--latest-branch-revision-max-processes=4"))
|
|
(getmail-idle-mailboxes '("Commits"))
|
|
(commits-getmail-retriever-configuration
|
|
(getmail-retriever-configuration
|
|
(type "SimpleIMAPSSLRetriever")
|
|
(server "imap.cbaines.net")
|
|
(port 993)
|
|
(username "patchwork")
|
|
(password-command
|
|
(list (file-append coreutils "/bin/cat")
|
|
"/etc/guix-data-service-commits-imap-password"))
|
|
(extra-parameters
|
|
'((mailboxes . ("Commits"))))))))
|
|
|
|
(service certbot-service-type
|
|
(certbot-configuration
|
|
(certificates
|
|
(list (certificate-configuration
|
|
(domains '("data.guix.gnu.org")))))
|
|
(email "mail@cbaines.net")
|
|
(webroot "/srv/http/data.guix.gnu.org")))
|
|
|
|
(service nginx-service-type %nginx-configuration))
|
|
|
|
(modify-services %base-services
|
|
(rottlog-service-type
|
|
config => (rottlog-configuration
|
|
(inherit config)
|
|
(rc-file
|
|
(plain-file
|
|
"rottlog-rc"
|
|
"
|
|
packer=gzip
|
|
compress=\"-9c\"
|
|
unpacker=gunzip
|
|
uncompress=\"-c\"
|
|
extension=\"gz\"
|
|
pager=\"less\"
|
|
packdir=\"/var/log\"
|
|
fromuser=\"root@data.guix.gnu.org\"
|
|
touser=\"mail@cbaines.net\"
|
|
notifempty=1
|
|
mail=\"sendmail -t\"
|
|
maxdepth=0
|
|
dir_own=root
|
|
dir_grp=root
|
|
dir_perm=0700
|
|
remove_missing
|
|
"))))
|
|
(guix-service-type
|
|
config => (guix-configuration
|
|
(inherit config)
|
|
(timeout (* 60 60 6))
|
|
(max-silent-time (* 60 60 2)) ; max 2 hours of silence
|
|
(extra-options '("--max-jobs=3"
|
|
"--cache-failures"))))))))
|