maintenance/hydra/nginx/berlin.scm

;; Nginx configuration for ci.guix.info

(use-modules (gnu services web))

;; TODO: these settings cannot currently expressed with Guix:

;; # This is a 72-core machine, but let's not use all of them for nginx.
;; worker_processes 32;
;;
;; error_log  /var/log/nginx/error.log error;
;; pcre_jit   on;
;;
;; events {
;;     worker_connections  1024;
;; }



(define* (le host #:optional privkey)
  (string-append "/etc/letsencrypt/live/"
                 host "/"
                 (if privkey "privkey" "fullchain")
                 ".pem"))

(define %publish-url "http://localhost:3000")

(define %tls-settings
  (list
   ;; Make sure SSL is disabled.
   "ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;"
   ;; Disable weak cipher suites.
   "ssl_ciphers         HIGH:!aNULL:!MD5;"
   "ssl_prefer_server_ciphers on;"

   ;; Use our own DH parameters created with:
   ;;    openssl dhparam -out dhparams.pem 2048
   ;; as suggested at <https://weakdh.org/sysadmin.html>.
   "ssl_dhparam         /etc/dhparams.pem;"))

(define %berlin-locations
  (list
   ;; Cuirass.
   (nginx-location-configuration
    (uri "/")
    (body (list "proxy_pass http://localhost:8081;")))

   (nginx-location-configuration
    (uri "/static")
    (body
     (list
      "proxy_pass http://localhost:8081;"
      ;; Let browsers cache this for a while.
      "expires 10d;"
      ;; Cache quite aggressively.
      "proxy_cache static;"
      "proxy_cache_valid 200 5d;"
      "proxy_cache_valid any 10m;"
      "proxy_ignore_client_abort on;")))

   (nginx-location-configuration
    (uri "/berlin.guixsd.org-export.pub")
    (body
     (list "root /var/www/guix;")))

   (nginx-location-configuration
    (uri "/nix-cache-info")
    (body
     (list
      (string-append
       "proxy_pass " %publish-url "/nix-cache-info;")
      ;; Cache this file since that's always the first thing we ask
      ;; for.
      "proxy_cache static;"
      "proxy_cache_valid 200 100d;"   ; cache hits for a looong time.
      "proxy_cache_valid any 5m;"     ; cache misses/others for 5 min.
      "proxy_ignore_client_abort on;"

      ;; We need to hide and ignore the Set-Cookie header to enable
      ;; caching.
      "proxy_hide_header    Set-Cookie;"
      "proxy_ignore_headers Set-Cookie;")))

   (nginx-location-configuration
    (uri "/nar/")
    (body
     (list
      (string-append "proxy_pass " %publish-url ";")
      "client_body_buffer_size 256k;"

      ;; Be more tolerant of delays when fetching a nar.
      "proxy_read_timeout 60s;"
      "proxy_send_timeout 60s;"

      ;; Enable caching for nar files, to avoid reconstructing and
      ;; recompressing archives.
      "proxy_cache nar;"
      "proxy_cache_valid 200 30d;" ; cache hits for 1 month
      "proxy_cache_valid 504 3m;"  ; timeout, when hydra.gnu.org is overloaded
      "proxy_cache_valid any 1h;"  ; cache misses/others for 1h.

      "proxy_ignore_client_abort on;"

      ;; Nars are already compressed.
      "gzip off;"

      ;; We need to hide and ignore the Set-Cookie header to enable
      ;; caching.
      "proxy_hide_header    Set-Cookie;"
      "proxy_ignore_headers Set-Cookie;"

      ;; Provide a 'content-length' header so that 'guix
      ;; substitute-binary' knows upfront how much it is downloading.
      ;; "add_header Content-Length $body_bytes_sent;"
      )))

   (nginx-location-configuration
    (uri "~ \\.narinfo$")
    (body
     (list
      ;; Since 'guix publish' has its own caching, and since it relies
      ;; on the atime of cached narinfos to determine whether a
      ;; narinfo can be removed from the cache, don't do any caching
      ;; here.
      (string-append "proxy_pass " %publish-url ";")

      ;; For HTTP pipelining.  This has a dramatic impact on
      ;; performance.
      "client_body_buffer_size 128k;"

      ;; Narinfos requests are short, serve many of them on a
      ;; connection.
      "keepalive_requests 600;"

      ;; Do not tolerate slowness of hydra.gnu.org when fetching
      ;; narinfos: better return 504 quickly than wait forever.
      "proxy_connect_timeout 2s;"
      "proxy_read_timeout 2s;"
      "proxy_send_timeout 2s;"

      ;; 'guix publish --ttl' produces a 'Cache-Control' header for
      ;; use by 'guix substitute'.  Let it through rather than use
      ;; nginx's "expire" directive since the expiration time defined
      ;; by 'guix publish' is the right one.
      "proxy_pass_header Cache-Control;"

      "proxy_ignore_client_abort on;"

      ;; We need to hide and ignore the Set-Cookie header to enable
      ;; caching.
      "proxy_hide_header    Set-Cookie;"
      "proxy_ignore_headers Set-Cookie;")))

   (nginx-location-configuration
    (uri "/log/")
    (body
     (list
      (string-append "proxy_pass " %publish-url ";")

      ;; Enable caching for build logs.
      "proxy_cache logs;"
      "proxy_cache_valid 200 60d;"      ; cache hits.
      "proxy_cache_valid 504 3m;" ; timeout, when hydra.gnu.org is overloaded
      "proxy_cache_valid any 1h;" ; cache misses/others.

      "proxy_ignore_client_abort on;"

      ;; We need to hide and ignore the Set-Cookie header to enable
      ;; caching.
      "proxy_hide_header    Set-Cookie;"
      "proxy_ignore_headers Set-Cookie;")))

   ;; Content-addressed files served by 'guix publish'.
   (nginx-location-configuration
    (uri "/file/")
    (body
     (list
      (string-append "proxy_pass " %publish-url ";")

      "proxy_cache cas;"
      "proxy_cache_valid 200 200d;"     ; cache hits
      "proxy_cache_valid any 5m;"       ; cache misses/others
      
      "proxy_ignore_client_abort on;")))

   ;; For use by Certbot.
   (nginx-location-configuration
    (uri "/.well-known")
    (body (list "root /var/www;")))))

(define %berlin-servers
  (list

   ;; Plain HTTP
   (nginx-server-configuration
    (listen '("80"))
    (server-name '("berlin.guixsd.org"
                   "ci.guix.info"
                   "ci.guix.gnu.org"))
    (locations %berlin-locations)
    (raw-content
     (list
      "access_log  /var/log/nginx/http.access.log;"
      "proxy_set_header X-Forwarded-Host $host;"
      "proxy_set_header X-Forwarded-Port $server_port;"
      "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;")))

   (nginx-server-configuration
    (listen '("80"))
    (server-name '("bootstrappable.org"
                   "www.bootstrappable.org"))
    (root "/home/rekado/bootstrappable.org")
    (raw-content
     (list
      "access_log /var/log/nginx/bootstrappable.access.log;")))

   (nginx-server-configuration
    (listen '("80"))
    (server-name '("guix.info"
                   "www.guix.info"
                   "guix.gnu.org"))
    (root "/home/rekado/guix.info")
    (raw-content
     (list
      "access_log /var/log/nginx/guix-info.access.log;")))

   (nginx-server-configuration
    (listen '("80"))
    (server-name '("issues.guix.info"
                   "issues.guix.gnu.org"))
    (root "/home/rekado/mumi/")
    (locations
     (list (nginx-location-configuration
            (uri "/")
            (body '("proxy_pass http://localhost:1234;")))))
    (raw-content
     (list
      "access_log /var/log/nginx/issues-guix-info.access.log;")))

   (nginx-server-configuration
    (listen '("80"))
    (server-name '("workflows.guix.info"
                   "workflow.guix.info"
                   "guixwl.org"
                   "www.guixwl.org"))
    (root "/home/rekado/gwl/")
    (locations
     (list (nginx-location-configuration
            (uri "/")
            (body '("proxy_pass http://localhost:5000;")))))
    (raw-content
     (list
      "access_log /var/log/nginx/workflows-guix-info.access.log;")))
        
   ;; HTTPS servers
   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("berlin.guixsd.org"
                   "ci.guix.info"
                   "ci.guix.gnu.org"))
    (ssl-certificate (le "berlin.guixsd.org"))
    (ssl-certificate-key (le "berlin.guixsd.org" 'key))
    (locations %berlin-locations)
    (raw-content
     (append
      %tls-settings
      (list
       "access_log  /var/log/nginx/https.access.log;"
       "proxy_set_header X-Forwarded-Host $host;"
       "proxy_set_header X-Forwarded-Port $server_port;"
       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"))))

   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("bootstrappable.org"
                   "www.bootstrappable.org"))
    (ssl-certificate (le "bootstrappable.org"))
    (ssl-certificate-key (le "bootstrappable.org" 'key))
    (root "/home/rekado/bootstrappable.org")
    (raw-content
     (append
      %tls-settings
      (list
       "access_log /var/log/nginx/bootstrappable.https.access.log;"))))

   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("guix.info"
                   "www.guix.info"))
    (ssl-certificate (le "guix.info"))
    (ssl-certificate-key (le "guix.info" 'key))
    (root "/home/rekado/guix.info")
    (raw-content
     (append
      %tls-settings
      (list
       "access_log /var/log/nginx/guix-info.https.access.log;"))))

   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("guix.gnu.org"))
    (ssl-certificate (le "guix.gnu.org"))
    (ssl-certificate-key (le "guix.gnu.org" 'key))
    (root "/home/rekado/guix.info")
    (raw-content
     (append
      %tls-settings
      (list
       "access_log /var/log/nginx/guix-gnu-org.https.access.log;"))))

   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("issues.guix.info"))
    (ssl-certificate (le "issues.guix.info"))
    (ssl-certificate-key (le "issues.guix.info" 'key))
    (root "/home/rekado/mumi/")
    (locations
     (list (nginx-location-configuration
            (uri "/")
            (body '("proxy_pass http://localhost:1234;")))))
    (raw-content
     (append
      %tls-settings
      (list
       "proxy_set_header X-Forwarded-Host $host;"
       "proxy_set_header X-Forwarded-Port $server_port;"
       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
       "proxy_connect_timeout       600;"
       "proxy_send_timeout          600;"
       "proxy_read_timeout          600;"
       "send_timeout                600;"
       "access_log /var/log/nginx/issues-guix-info.https.access.log;"))))

   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("issues.guix.gnu.org"))
    (ssl-certificate (le "issues.guix.gnu.org"))
    (ssl-certificate-key (le "issues.guix.gnu.org" 'key))
    (root "/home/rekado/mumi/")
    (locations
     (list (nginx-location-configuration
            (uri "/")
            (body '("proxy_pass http://localhost:1234;")))))
    (raw-content
     (append
      %tls-settings
      (list
       "proxy_set_header X-Forwarded-Host $host;"
       "proxy_set_header X-Forwarded-Port $server_port;"
       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
       "proxy_connect_timeout       600;"
       "proxy_send_timeout          600;"
       "proxy_read_timeout          600;"
       "send_timeout                600;"
       "access_log /var/log/nginx/issues-guix-gnu-org.https.access.log;"))))

   (nginx-server-configuration
    (listen '("443 ssl"))
    (server-name '("workflows.guix.info"
                   "workflow.guix.info"
                   "guixwl.org"
                   "www.guixwl.org"))
    (ssl-certificate (le "workflows.guix.info"))
    (ssl-certificate-key (le "workflows.guix.info" 'key))
    (root "/home/rekado/gwl/")
    (locations
     (list (nginx-location-configuration
            (uri "/")
            (body '("proxy_pass http://localhost:5000;")))))
    (raw-content
     (append
      %tls-settings
      (list
       "proxy_set_header X-Forwarded-Host $host;"
       "proxy_set_header X-Forwarded-Port $server_port;"
       "proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;"
       "proxy_connect_timeout       600;"
       "proxy_send_timeout          600;"
       "proxy_read_timeout          600;"
       "send_timeout                600;"
       "access_log /var/log/nginx/workflows-guix-info.https.access.log;"))))))

(define %extra-content
  (list
   "default_type  application/octet-stream;"
   "sendfile        on;"

   ;; Maximum chunk size to send.  Partly this is a workaround for
   ;; <http://bugs.gnu.org/19939>, but also the nginx docs mention that
   ;; "Without the limit, one fast connection may seize the worker
   ;; process entirely."
   ;;  <http://nginx.org/en/docs/http/ngx_http_core_module#sendfile_max_chunk>
   "sendfile_max_chunk 1m;"

   "keepalive_timeout  65;"

   ;; Use HTTP 1.1 to talk to the backend so we benefit from keep-alive
   ;; connections and chunked transfer encoding.  The latter allows us to
   ;; make sure we do not cache partial downloads.
   "proxy_http_version 1.1;"

   ;; The 'inactive' parameter for caching is not very useful in our
   ;; case: all that matters is that LRU sweeping happens when 'max_size'
   ;; is hit.

   ;; cache for nar files
   "proxy_cache_path /var/cache/nginx/nar"
   "     levels=2"
   "     inactive=8d"	    ; inactive keys removed after 8d
   "     keys_zone=nar:4m"  ; nar cache meta data: ~32K keys
   "     max_size=10g;"     ; total cache data size max

   ;; cache for content-addressed files
   "proxy_cache_path /var/cache/nginx/cas"
   "     levels=2"
   "     inactive=180d"	    ; inactive keys removed after 180d
   "     keys_zone=cas:8m"  ; nar cache meta data: ~64K keys
   "     max_size=50g;"         ; total cache data size max

   ;; cache for build logs
   "proxy_cache_path /var/cache/nginx/logs"
   "     levels=2"
   "     inactive=60d"          ; inactive keys removed after 60d
   "     keys_zone=logs:8m"     ; narinfo meta data: ~64K keys
   "     max_size=4g;"          ; total cache data size max

   ;; cache for static data
   "proxy_cache_path /var/cache/nginx/static"
   "     levels=1"
   "     inactive=10d"	       ; inactive keys removed after 10d
   "     keys_zone=static:1m"   ; nar cache meta data: ~8K keys
   "     max_size=200m;"        ; total cache data size max

   ;; If Hydra cannot honor these delays, then something is wrong and
   ;; we'd better drop the connection and return 504.
   "proxy_connect_timeout 7s;"
   "proxy_read_timeout 10s;"
   "proxy_send_timeout 10s;"

   ;; Cache timeouts for a little while to avoid increasing pressure.
   "proxy_cache_valid 504 30s;"))

(define %nginx-configuration
  (nginx-configuration
   (server-blocks %berlin-servers)
   (extra-content
    (string-join %extra-content "\n"))))