2016-07-16 18:16:39 +02:00
|
|
|
;;; database.scm -- store evaluation and build results
|
2017-01-09 01:29:48 +01:00
|
|
|
;;; Copyright © 2016, 2017 Mathieu Lirzin <mthl@gnu.org>
|
2020-06-10 15:58:11 +02:00
|
|
|
;;; Copyright © 2017, 2020 Mathieu Othacehe <m.othacehe@gmail.com>
|
2020-02-23 23:25:23 +01:00
|
|
|
;;; Copyright © 2018, 2020 Ludovic Courtès <ludo@gnu.org>
|
2018-07-07 00:31:14 +02:00
|
|
|
;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
;;; Copyright © 2018 Tatiana Sholokhova <tanja201396@gmail.com>
|
2020-04-15 19:50:25 +02:00
|
|
|
;;; Copyright © 2019, 2020 Ricardo Wurmus <rekado@elephly.net>
|
2016-06-26 15:13:31 +02:00
|
|
|
;;;
|
|
|
|
;;; This file is part of Cuirass.
|
|
|
|
;;;
|
2016-07-23 12:10:04 +02:00
|
|
|
;;; Cuirass is free software: you can redistribute it and/or modify
|
|
|
|
;;; it under the terms of the GNU General Public License as published by
|
|
|
|
;;; the Free Software Foundation, either version 3 of the License, or
|
|
|
|
;;; (at your option) any later version.
|
2016-06-26 15:13:31 +02:00
|
|
|
;;;
|
2016-07-23 12:10:04 +02:00
|
|
|
;;; Cuirass is distributed in the hope that it will be useful,
|
|
|
|
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
|
2016-06-26 15:13:31 +02:00
|
|
|
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
;;; GNU General Public License for more details.
|
|
|
|
;;;
|
|
|
|
;;; You should have received a copy of the GNU General Public License
|
|
|
|
;;; along with Cuirass. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
(define-module (cuirass database)
|
2020-08-01 11:56:43 +02:00
|
|
|
#:use-module (cuirass logging)
|
2016-06-26 18:40:31 +02:00
|
|
|
#:use-module (cuirass config)
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
#:use-module (cuirass remote)
|
2016-07-23 22:05:50 +02:00
|
|
|
#:use-module (cuirass utils)
|
2021-01-05 10:20:34 +01:00
|
|
|
#:use-module (squee)
|
2016-07-23 22:05:50 +02:00
|
|
|
#:use-module (ice-9 match)
|
2018-01-24 23:40:07 +01:00
|
|
|
#:use-module (ice-9 format)
|
2018-07-07 00:31:14 +02:00
|
|
|
#:use-module (ice-9 ftw)
|
2016-07-16 18:16:39 +02:00
|
|
|
#:use-module (ice-9 rdelim)
|
2018-07-07 00:31:14 +02:00
|
|
|
#:use-module (ice-9 regex)
|
2020-04-17 14:23:18 +02:00
|
|
|
#:use-module (ice-9 threads)
|
2017-07-31 19:25:28 +02:00
|
|
|
#:use-module (srfi srfi-1)
|
2018-01-23 18:15:42 +01:00
|
|
|
#:use-module (srfi srfi-19)
|
2018-02-19 16:30:07 +01:00
|
|
|
#:use-module (srfi srfi-26)
|
2020-09-22 15:17:31 +02:00
|
|
|
#:use-module (system foreign)
|
|
|
|
#:use-module (rnrs bytevectors)
|
2016-06-26 18:40:31 +02:00
|
|
|
#:export (;; Procedures.
|
|
|
|
db-init
|
2016-06-26 15:13:31 +02:00
|
|
|
db-open
|
|
|
|
db-close
|
2021-01-05 10:20:34 +01:00
|
|
|
exec-query/bind-params
|
|
|
|
expect-one-row
|
|
|
|
read-sql-file
|
|
|
|
db-add-input
|
|
|
|
db-add-checkout
|
2016-07-23 17:00:38 +02:00
|
|
|
db-add-specification
|
2019-10-30 09:19:28 +01:00
|
|
|
db-remove-specification
|
2021-01-05 10:20:34 +01:00
|
|
|
db-get-inputs
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
db-get-specification
|
2016-07-23 22:05:50 +02:00
|
|
|
db-get-specifications
|
2020-09-10 14:25:19 +02:00
|
|
|
evaluation-status
|
2016-06-26 15:13:31 +02:00
|
|
|
db-add-evaluation
|
2020-09-10 14:25:19 +02:00
|
|
|
db-abort-pending-evaluations
|
|
|
|
db-set-evaluation-status
|
2020-09-06 13:03:08 +02:00
|
|
|
db-set-evaluation-time
|
2018-01-23 17:37:08 +01:00
|
|
|
build-status
|
2021-01-05 10:20:34 +01:00
|
|
|
db-add-output
|
2016-07-25 20:31:06 +02:00
|
|
|
db-add-build
|
2020-06-10 15:58:11 +02:00
|
|
|
db-add-build-product
|
2021-01-05 10:20:34 +01:00
|
|
|
db-get-output
|
|
|
|
db-get-outputs
|
|
|
|
db-get-time-since-previous-build
|
2020-09-29 14:34:14 +02:00
|
|
|
db-register-builds
|
2018-01-23 18:15:42 +01:00
|
|
|
db-update-build-status!
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
db-update-build-worker!
|
2021-01-05 10:20:34 +01:00
|
|
|
db-get-build-products
|
2019-05-17 22:33:27 +02:00
|
|
|
db-get-builds-by-search
|
2021-01-05 10:20:34 +01:00
|
|
|
db-get-builds
|
|
|
|
db-get-build
|
2019-10-21 23:43:16 +02:00
|
|
|
db-get-events
|
|
|
|
db-delete-events-with-ids-<=-to
|
2021-01-05 10:20:34 +01:00
|
|
|
db-get-pending-derivations
|
|
|
|
db-get-checkouts
|
2020-12-07 15:34:29 +01:00
|
|
|
db-get-evaluation
|
2018-04-19 11:17:42 +02:00
|
|
|
db-get-evaluations
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
db-get-evaluations-build-summary
|
|
|
|
db-get-evaluations-id-min
|
|
|
|
db-get-evaluations-id-max
|
2021-01-05 10:20:34 +01:00
|
|
|
db-get-evaluation-summary
|
|
|
|
db-get-builds-query-min
|
|
|
|
db-get-builds-query-max
|
|
|
|
db-get-builds-min
|
|
|
|
db-get-builds-max
|
2018-08-05 21:25:37 +02:00
|
|
|
db-get-evaluation-specification
|
2020-06-10 15:58:11 +02:00
|
|
|
db-get-build-product-path
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
db-add-worker
|
|
|
|
db-get-workers
|
|
|
|
db-clear-workers
|
2021-01-05 10:20:34 +01:00
|
|
|
db-clear-build-queue
|
2016-06-26 18:40:31 +02:00
|
|
|
;; Parameters.
|
2021-01-15 14:46:50 +01:00
|
|
|
%create-database?
|
2016-06-26 18:40:31 +02:00
|
|
|
%package-database
|
2016-07-23 17:00:38 +02:00
|
|
|
%package-schema-file
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
%db-channel
|
2019-10-21 23:43:16 +02:00
|
|
|
%record-events?
|
2016-06-26 18:40:31 +02:00
|
|
|
;; Macros.
|
2021-01-05 10:20:34 +01:00
|
|
|
exec-query/bind
|
2020-09-30 11:56:04 +02:00
|
|
|
with-database
|
2021-01-05 10:20:34 +01:00
|
|
|
with-db-worker-thread))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
;; Maximum priority for a Build or Specification.
|
|
|
|
(define max-priority 9)
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define (%exec-query db query args)
|
|
|
|
(exec-query db query args))
|
|
|
|
|
|
|
|
(define (normalize obj)
|
|
|
|
(if (string? obj)
|
|
|
|
obj
|
|
|
|
(and obj (object->string obj))))
|
|
|
|
|
|
|
|
(define-syntax %exec-query/bind
|
2018-02-08 17:31:39 +01:00
|
|
|
(lambda (s)
|
2021-01-05 10:20:34 +01:00
|
|
|
;; Expand to an 'exec-query' call where the query string has
|
2018-02-08 17:31:39 +01:00
|
|
|
;; interspersed question marks and the argument list is separate.
|
|
|
|
(define (string-literal? s)
|
|
|
|
(string? (syntax->datum s)))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define (interleave a b)
|
|
|
|
(if (null? b)
|
|
|
|
(list (car a))
|
|
|
|
`(,(car a) ,(car b) ,@(interleave (cdr a) (cdr b)))))
|
|
|
|
|
|
|
|
(define (interleave-arguments str)
|
|
|
|
(string-join
|
|
|
|
(interleave str
|
|
|
|
(map (lambda (i)
|
|
|
|
(string-append "$"
|
|
|
|
(number->string (1+ i))))
|
|
|
|
(iota (1- (length str)))))
|
|
|
|
" "))
|
|
|
|
|
2018-02-08 17:31:39 +01:00
|
|
|
(syntax-case s ()
|
|
|
|
((_ db (bindings ...) tail str arg rest ...)
|
2021-01-05 10:20:34 +01:00
|
|
|
#'(%exec-query/bind db
|
2018-02-08 17:31:39 +01:00
|
|
|
(bindings ... (str arg))
|
|
|
|
tail
|
|
|
|
rest ...))
|
|
|
|
((_ db (bindings ...) tail str)
|
2021-01-05 10:20:34 +01:00
|
|
|
#'(%exec-query/bind db (bindings ...) str))
|
2018-02-08 17:31:39 +01:00
|
|
|
((_ db ((strings args) ...) tail)
|
|
|
|
;; Optimized case: only string literals.
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-syntax ((query
|
|
|
|
(interleave-arguments
|
|
|
|
(append (syntax->datum #'(strings ...))
|
|
|
|
(list (syntax->datum #'tail))))))
|
|
|
|
#'(%exec-query db query (map normalize (list args ...))))))))
|
|
|
|
|
|
|
|
(define-syntax-rule (exec-query/bind db query args ...)
|
|
|
|
"Execute the specific QUERY with the given ARGS. Uses of 'exec-query/bind'
|
2018-02-08 17:31:39 +01:00
|
|
|
typically look like this:
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query/bind db \"SELECT * FROM Foo WHERE x = \" x \"AND Y=\" y \";\")
|
2018-02-08 17:31:39 +01:00
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
References to variables 'x' and 'y' here are replaced by $1 and $2 in the
|
|
|
|
SQL query.
|
2018-02-08 17:31:39 +01:00
|
|
|
|
|
|
|
This ensures that (1) SQL injection is impossible, and (2) the number of
|
2021-01-05 10:20:34 +01:00
|
|
|
parameters matches the number of arguments to bind."
|
|
|
|
(%exec-query/bind db () "" query args ...))
|
|
|
|
|
|
|
|
(define (exec-query/bind-params db query params)
|
|
|
|
(define param-regex
|
|
|
|
(make-regexp ":[a-zA-Z]+"))
|
|
|
|
|
|
|
|
(define (argument-indexes arguments)
|
|
|
|
(let loop ((res '())
|
|
|
|
(bindings '())
|
|
|
|
(counter 1)
|
|
|
|
(arguments arguments))
|
|
|
|
(if (null? arguments)
|
|
|
|
(reverse res)
|
|
|
|
(let* ((arg (car arguments))
|
|
|
|
(index (assoc-ref bindings arg)))
|
|
|
|
(if index
|
|
|
|
(loop (cons index res)
|
|
|
|
bindings
|
|
|
|
counter
|
|
|
|
(cdr arguments))
|
|
|
|
(loop (cons counter res)
|
|
|
|
`((,arg . ,counter) ,@bindings)
|
|
|
|
(1+ counter)
|
|
|
|
(cdr arguments)))))))
|
|
|
|
|
|
|
|
(let* ((args
|
|
|
|
(reverse
|
|
|
|
(fold-matches param-regex query
|
|
|
|
'() (lambda (m p)
|
|
|
|
(cons (match:substring m) p)))))
|
|
|
|
(indexes (argument-indexes args))
|
|
|
|
(proc (lambda (m)
|
|
|
|
(let ((index (car indexes)))
|
|
|
|
(set! indexes (cdr indexes))
|
|
|
|
(string-append "$" (number->string index)))))
|
|
|
|
(query (regexp-substitute/global #f param-regex query
|
|
|
|
'pre proc 'post))
|
|
|
|
(params (map (lambda (arg)
|
|
|
|
(let ((symbol
|
|
|
|
(symbol->keyword
|
|
|
|
(string->symbol (substring arg 1)))))
|
|
|
|
(assoc-ref params symbol)))
|
|
|
|
(delete-duplicates args))))
|
|
|
|
(exec-query db query (map normalize params))))
|
2018-11-16 21:47:18 +01:00
|
|
|
|
2021-01-15 14:46:50 +01:00
|
|
|
(define %create-database?
|
|
|
|
(make-parameter #f))
|
|
|
|
|
2016-06-26 18:40:31 +02:00
|
|
|
(define %package-database
|
2021-01-05 10:20:34 +01:00
|
|
|
(make-parameter #f))
|
2016-06-26 18:40:31 +02:00
|
|
|
|
2016-07-16 18:16:39 +02:00
|
|
|
(define %package-schema-file
|
|
|
|
;; Define to the database schema file of this package.
|
|
|
|
(make-parameter (string-append (or (getenv "CUIRASS_DATADIR")
|
|
|
|
(string-append %datadir "/" %package))
|
|
|
|
"/schema.sql")))
|
|
|
|
|
2018-07-07 00:31:14 +02:00
|
|
|
(define %package-sql-dir
|
|
|
|
;; Define to the directory containing the SQL files.
|
|
|
|
(make-parameter (string-append (or (getenv "CUIRASS_DATADIR")
|
|
|
|
(string-append %datadir "/" %package))
|
|
|
|
"/sql")))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define %db-channel
|
|
|
|
(make-parameter #f))
|
|
|
|
|
2019-10-21 23:43:16 +02:00
|
|
|
(define %record-events?
|
|
|
|
(make-parameter #f))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define-syntax-rule (with-database body ...)
|
|
|
|
"Run BODY with %DB-CHANNEL being dynamically bound to a channel providing a
|
|
|
|
worker thread that allows database operations to run without interfering with
|
|
|
|
fibers."
|
|
|
|
(parameterize ((%db-channel
|
|
|
|
(make-worker-thread-channel
|
|
|
|
(lambda ()
|
|
|
|
(list (db-open)))
|
|
|
|
#:parallelism
|
|
|
|
(min (current-processor-count) 8))))
|
|
|
|
body ...))
|
2020-12-07 17:15:22 +01:00
|
|
|
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(define-syntax-rule (with-db-worker-thread db exp ...)
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
"Evaluate EXP... in the critical section corresponding to %DB-CHANNEL.
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
DB is bound to the argument of that critical section: the database connection.
|
|
|
|
This must only be used for reading queries, i.e SELECT queries."
|
2020-10-03 10:14:33 +02:00
|
|
|
(let ((send-timeout 2)
|
|
|
|
(receive-timeout 5)
|
2020-10-03 08:26:36 +02:00
|
|
|
(caller-name (frame-procedure-name
|
|
|
|
(stack-ref (make-stack #t) 1))))
|
2020-08-01 11:56:43 +02:00
|
|
|
(call-with-worker-thread
|
|
|
|
(%db-channel)
|
|
|
|
(lambda (db) exp ...)
|
2020-10-03 10:14:33 +02:00
|
|
|
#:send-timeout send-timeout
|
|
|
|
#:send-timeout-proc
|
|
|
|
(lambda ()
|
|
|
|
(log-message
|
|
|
|
(format #f "No available database workers for ~a seconds."
|
|
|
|
(number->string send-timeout))))
|
|
|
|
#:receive-timeout receive-timeout
|
|
|
|
#:receive-timeout-proc
|
2020-08-01 11:56:43 +02:00
|
|
|
(lambda ()
|
|
|
|
(log-message
|
2020-10-03 08:26:36 +02:00
|
|
|
(format #f "Database worker unresponsive for ~a seconds (~a)."
|
2020-10-03 10:14:33 +02:00
|
|
|
(number->string receive-timeout)
|
2020-10-03 08:26:36 +02:00
|
|
|
caller-name))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2016-07-16 18:16:39 +02:00
|
|
|
(define (read-sql-file file-name)
|
|
|
|
"Return a list of string containing SQL instructions from FILE-NAME."
|
|
|
|
(call-with-input-file file-name
|
2017-07-06 10:03:16 +02:00
|
|
|
(lambda (port)
|
2016-07-16 18:16:39 +02:00
|
|
|
(let loop ((insts '()))
|
|
|
|
(let ((inst (read-delimited ";" port 'concat)))
|
|
|
|
(if (or (eof-object? inst)
|
|
|
|
;; Don't cons the spaces after the last instructions.
|
|
|
|
(string-every char-whitespace? inst))
|
|
|
|
(reverse! insts)
|
|
|
|
(loop (cons inst insts))))))))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define (expect-one-row rows)
|
|
|
|
"Several SQL queries expect one result, or zero if not found. This gets rid
|
|
|
|
of the list, and returns #f when there is no result."
|
|
|
|
(match rows
|
|
|
|
((row) row)
|
|
|
|
(() #f)))
|
2018-01-26 18:20:30 +01:00
|
|
|
|
2018-07-07 00:31:14 +02:00
|
|
|
(define (db-load db schema)
|
|
|
|
"Evaluate the file SCHEMA, which may contain SQL queries, into DB."
|
2021-01-05 10:20:34 +01:00
|
|
|
(for-each (cut exec-query db <>)
|
2018-07-07 00:31:14 +02:00
|
|
|
(read-sql-file schema)))
|
|
|
|
|
|
|
|
(define (db-schema-version db)
|
2021-01-05 10:20:34 +01:00
|
|
|
(catch 'psql-query-error
|
|
|
|
(lambda ()
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query db "SELECT version FROM SchemaVersion"))
|
|
|
|
((version) (string->number version))))
|
|
|
|
(lambda _ #f)))
|
2018-07-07 00:31:14 +02:00
|
|
|
|
|
|
|
(define (db-set-schema-version db version)
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query db "DELETE FROM SchemaVersion")
|
|
|
|
(exec-query/bind db "INSERT INTO SchemaVersion (version) VALUES
|
|
|
|
(" version ")"))
|
2018-07-07 00:31:14 +02:00
|
|
|
|
|
|
|
(define (latest-db-schema-version)
|
|
|
|
"Return the version to which the schema should be upgraded, based on the
|
|
|
|
upgrade-n.sql files, or 0 if there are no such files."
|
|
|
|
(reduce max 0
|
|
|
|
(map (compose string->number (cut match:substring <> 1))
|
|
|
|
(filter-map (cut string-match "^upgrade-([0-9]+)\\.sql$" <>)
|
|
|
|
(or (scandir (%package-sql-dir)) '())))))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define* (db-init db
|
|
|
|
#:key
|
|
|
|
(schema (%package-schema-file)))
|
2016-06-26 18:40:31 +02:00
|
|
|
"Open the database to store and read jobs and builds informations. Return a
|
|
|
|
database object."
|
2021-01-05 10:20:34 +01:00
|
|
|
(db-load db schema)
|
|
|
|
(db-set-schema-version db (latest-db-schema-version))
|
|
|
|
db)
|
2016-06-26 15:13:31 +02:00
|
|
|
|
2018-07-07 00:31:14 +02:00
|
|
|
(define (schema-upgrade-file version)
|
|
|
|
"Return the file containing the SQL instructions that upgrade the schema
|
|
|
|
from VERSION-1 to VERSION."
|
|
|
|
(in-vicinity (%package-sql-dir) (format #f "upgrade-~a.sql" version)))
|
|
|
|
|
|
|
|
(define (db-upgrade db)
|
|
|
|
"Upgrade database DB based on its current version and the available
|
|
|
|
upgrade-n.sql files."
|
|
|
|
(for-each (lambda (version)
|
|
|
|
(db-load db (schema-upgrade-file version))
|
|
|
|
(db-set-schema-version db version))
|
|
|
|
(let ((current (db-schema-version db)))
|
|
|
|
(iota (- (latest-db-schema-version) current) (1+ current))))
|
|
|
|
db)
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define* (db-open #:key
|
|
|
|
(database (%package-database)))
|
2016-06-26 18:40:31 +02:00
|
|
|
"Open database to store or read jobs and builds informations. Return a
|
|
|
|
database object."
|
2021-01-05 10:20:34 +01:00
|
|
|
(let* ((param (or database
|
|
|
|
(format #f "dbname=~a host=~a"
|
|
|
|
(getenv "CUIRASS_DATABASE")
|
|
|
|
(getenv "CUIRASS_HOST"))))
|
|
|
|
(db (connect-to-postgres-paramstring param)))
|
2021-01-15 14:46:50 +01:00
|
|
|
(when (%create-database?)
|
|
|
|
(match (db-schema-version db)
|
|
|
|
(#f
|
|
|
|
(db-init db))
|
|
|
|
(else
|
|
|
|
(db-upgrade db))))
|
|
|
|
db))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
|
|
|
(define (db-close db)
|
|
|
|
"Close database object DB."
|
2021-01-05 10:20:34 +01:00
|
|
|
(pg-conn-finish db))
|
2018-09-30 17:26:12 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-add-input spec-name input)
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "\
|
|
|
|
INSERT INTO Inputs (specification, name, url, load_path, branch, \
|
Add support for multiple inputs.
* Makefile.am (dist_sql_DATA): Add src/sql/upgrade-1.sql.
* bin/cuirass.in (show-help, %options, main): Remove the LOAD-PATH option that
was used afterwards as %GUIX-PACKAGE-PATH.
* bin/evaluate.in (absolutize, input-checkout, spec-source, spec-load-path,
spec-package-path, format-checkouts): New procedures.
(%not-colon): Remove variable.
(main): Take the load path, package path and PROC from the checkouts that
result from the inputs. Format the checkouts before sending them to the
procedure. Remove the LOAD-PATH argument.
* doc/cuirass.texi (Overview, Database schema): Document the changes.
* examples/{guix-jobs.scm, hello-git.scm, hello-singleton.scm,
hello-subset.scm, random.scm}: Adapt to the new specification format.
* examples/guix-track-git.scm (package->spec): Rename to PACKAGE->INPUT.
(package->git-tracked): Replace FETCH-REPOSITORY with FETCH-INPUT and handle
the new format of its return value.
* examples/random-jobs.scm (make-random-jobs): Rename RANDOM to CHECKOUT.
Rename the checkout from 'random (which is a specification) to 'cuirass (which
is a checkout resulting from an input).
* src/cuirass/base.scm (fetch-repository): Rename to fetch-input. Rename SPEC
to INPUT. Return a checkout object instead of returning two values.
(evaluate): Take a list of CHECKOUTS and COMMITS as arguments, instead of
SOURCE. Remove TOKENIZE and LOAD-PATH. Pass the CHECKOUTS instead of the
SOURCE to "evaluate". Remove %GUIX-PACKAGE-PATH. Build the EVAL object
instead of getting it from "evaluate".
(compile?, fetch-inputs, compile-checkouts): New procedures.
(process-specs): Fetch all inputs instead of only fetching one repository.
The result of that fetching operation is a list of CHECKOUTS whose COMMITS are
used as a STAMP.
(%guix-package-path, set-guix-package-path): Remove them.
* src/cuirass/database.scm (db-add-input, db-get-inputs): New procedures.
(db-add-specification, db-get-specifications): Adapt to the new specification
format. Add/get all inputs as well.
(db-add-evaluation): Rename REVISION to COMMITS. Store COMMITS as space
separated commit hashes.
(db-get-builds): Rename REPO_NAME to NAME.
(db-get-stamp): Rename COMMIT to STAMP. Return #f when there is no STAMP.
(db-add-stamp): Rename COMMIT to STAMP. Deal with DB-GET-STAMP's new return
value.
(db-get-evaluations): Rename REVISION to COMMITS. Tokenize COMMITS.
* src/cuirass/utils.scm (%non-blocking): Export it.
* src/schema.sql (Inputs): New table that refers to the Specifications table.
(Specifications): Move input related fields to the Inputs table. Rename
REPO_NAME to NAME. Rename ARGUMENTS to PROC_ARGS. Rename FILE to PROC_FILE.
Add LOAD_PATH_INPUTS, PACKAGE_PATH_INPUTS and PROC_INPUT fields that refer to
the Inputs table.
(Stamps): Rename REPO_NAME to NAME.
(Evaluations): Rename REPO_NAME to NAME. Rename REVISION to COMMITS.
(Specifications_index): Replace with Inputs_index.
* src/sql/upgrade-1.sql: New file.
* tests/database.scm (example-spec, make-dummy-eval, sqlite-exec): Adapt to
the new specifications format. Rename REVISION to COMMITS.
* tests/http.scm (evaluations-query-result, fill-db): Idem.
2018-06-26 11:18:23 +02:00
|
|
|
tag, revision, no_compile_p) VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
spec-name ", "
|
|
|
|
(assq-ref input #:name) ", "
|
|
|
|
(assq-ref input #:url) ", "
|
|
|
|
(assq-ref input #:load-path) ", "
|
|
|
|
(assq-ref input #:branch) ", "
|
|
|
|
(assq-ref input #:tag) ", "
|
|
|
|
(assq-ref input #:commit) ", "
|
|
|
|
(if (assq-ref input #:no-compile?) 1 0) ")
|
|
|
|
ON CONFLICT ON CONSTRAINT inputs_pkey DO NOTHING;")))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(define (db-add-checkout spec-name eval-id checkout)
|
|
|
|
"Insert CHECKOUT associated with SPEC-NAME and EVAL-ID. If a checkout with
|
|
|
|
the same revision already exists for SPEC-NAME, return #f."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "\
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
INSERT INTO Checkouts (specification, revision, evaluation, input,
|
2020-09-06 13:03:08 +02:00
|
|
|
directory, timestamp) VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
spec-name ", "
|
|
|
|
(assq-ref checkout #:commit) ", "
|
|
|
|
eval-id ", "
|
|
|
|
(assq-ref checkout #:input) ", "
|
|
|
|
(assq-ref checkout #:directory) ", "
|
|
|
|
(or (assq-ref checkout #:timestamp) 0) ")
|
|
|
|
ON CONFLICT ON CONSTRAINT checkouts_pkey DO NOTHING
|
|
|
|
RETURNING (specification, revision);"))
|
|
|
|
(x x)
|
|
|
|
(() #f))))
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-add-specification spec)
|
|
|
|
"Store SPEC in database the database. SPEC inputs are stored in the INPUTS
|
|
|
|
table."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "\
|
|
|
|
INSERT INTO Specifications (name, load_path_inputs, \
|
2020-06-10 15:58:11 +02:00
|
|
|
package_path_inputs, proc_input, proc_file, proc, proc_args, \
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
build_outputs, priority) \
|
2018-02-08 17:31:39 +01:00
|
|
|
VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
(assq-ref spec #:name) ", "
|
|
|
|
(assq-ref spec #:load-path-inputs) ", "
|
|
|
|
(assq-ref spec #:package-path-inputs) ", "
|
|
|
|
(assq-ref spec #:proc-input) ", "
|
|
|
|
(assq-ref spec #:proc-file) ", "
|
|
|
|
(symbol->string (assq-ref spec #:proc)) ", "
|
|
|
|
(assq-ref spec #:proc-args) ", "
|
|
|
|
(assq-ref spec #:build-outputs) ", "
|
|
|
|
(or (assq-ref spec #:priority) max-priority) ")
|
|
|
|
ON CONFLICT ON CONSTRAINT specifications_pkey DO NOTHING
|
|
|
|
RETURNING name;"))
|
|
|
|
((name)
|
|
|
|
(for-each (lambda (input)
|
|
|
|
(db-add-input (assq-ref spec #:name) input))
|
|
|
|
(assq-ref spec #:inputs))
|
|
|
|
name)
|
|
|
|
(else #f))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2019-10-30 09:19:28 +01:00
|
|
|
(define (db-remove-specification name)
|
|
|
|
"Remove the specification matching NAME from the database and its inputs."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query db "BEGIN TRANSACTION;")
|
|
|
|
(exec-query/bind db "\
|
2019-10-30 09:19:28 +01:00
|
|
|
DELETE FROM Inputs WHERE specification=" name ";")
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query/bind db "\
|
2019-10-30 09:19:28 +01:00
|
|
|
DELETE FROM Specifications WHERE name=" name ";")
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query db "COMMIT;")))
|
2019-10-30 09:19:28 +01:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-inputs spec-name)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let loop ((rows (exec-query/bind
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
db "SELECT * FROM Inputs WHERE specification="
|
2021-01-05 10:20:34 +01:00
|
|
|
spec-name "ORDER BY name;"))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(inputs '()))
|
|
|
|
(match rows
|
2021-01-05 10:20:34 +01:00
|
|
|
(() (reverse inputs))
|
|
|
|
(((specification name url load-path branch tag revision no-compile-p)
|
|
|
|
. rest)
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(loop rest
|
|
|
|
(cons `((#:name . ,name)
|
|
|
|
(#:url . ,url)
|
|
|
|
(#:load-path . ,load-path)
|
|
|
|
(#:branch . ,branch)
|
|
|
|
(#:tag . ,tag)
|
|
|
|
(#:commit . ,revision)
|
2021-01-05 10:20:34 +01:00
|
|
|
(#:no-compile? . ,(positive?
|
|
|
|
(string->number no-compile-p))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
inputs)))))))
|
|
|
|
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
(define (db-get-specification name)
|
|
|
|
"Retrieve a specification in the database with the given NAME."
|
2021-01-05 10:20:34 +01:00
|
|
|
(expect-one-row (db-get-specifications name)))
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
|
|
|
|
(define* (db-get-specifications #:optional name)
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(let loop
|
|
|
|
((rows (if name
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query/bind db "
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
SELECT * FROM Specifications WHERE name =" name ";")
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query db "
|
|
|
|
SELECT * FROM Specifications ORDER BY name ASC;")))
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
(specs '()))
|
2021-01-05 10:20:34 +01:00
|
|
|
(match rows
|
|
|
|
(() (reverse specs))
|
|
|
|
(((name load-path-inputs package-path-inputs proc-input proc-file proc
|
|
|
|
proc-args build-outputs priority)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:name . ,name)
|
|
|
|
(#:load-path-inputs .
|
|
|
|
,(with-input-from-string load-path-inputs read))
|
|
|
|
(#:package-path-inputs .
|
|
|
|
,(with-input-from-string package-path-inputs read))
|
|
|
|
(#:proc-input . ,proc-input)
|
|
|
|
(#:proc-file . ,proc-file)
|
|
|
|
(#:proc . ,(with-input-from-string proc read))
|
|
|
|
(#:proc-args . ,(with-input-from-string proc-args read))
|
|
|
|
(#:inputs . ,(db-get-inputs name))
|
|
|
|
(#:build-outputs .
|
|
|
|
,(with-input-from-string build-outputs read))
|
|
|
|
(#:priority . ,(string->number priority)))
|
|
|
|
specs)))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-09-10 14:25:19 +02:00
|
|
|
(define-enumeration evaluation-status
|
|
|
|
(started -1)
|
|
|
|
(succeeded 0)
|
|
|
|
(failed 1)
|
|
|
|
(aborted 2))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define (db-add-event type timestamp details)
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(when (%record-events?)
|
|
|
|
(exec-query/bind db "\
|
|
|
|
INSERT INTO Events (type, timestamp, event_json) VALUES ("
|
|
|
|
(symbol->string type) ", "
|
|
|
|
timestamp ", "
|
|
|
|
(object->json-string details)
|
|
|
|
");")
|
|
|
|
#t)))
|
|
|
|
|
2020-09-06 13:03:08 +02:00
|
|
|
(define* (db-add-evaluation spec-name checkouts
|
|
|
|
#:key
|
|
|
|
(checkouttime 0)
|
|
|
|
(evaltime 0)
|
|
|
|
timestamp)
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
"Add a new evaluation for SPEC-NAME only if one of the CHECKOUTS is new.
|
|
|
|
Otherwise, return #f."
|
2020-09-06 13:03:08 +02:00
|
|
|
(define now
|
|
|
|
(or timestamp (time-second (current-time time-utc))))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query db "BEGIN TRANSACTION;")
|
|
|
|
(let* ((eval-id
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "\
|
|
|
|
INSERT INTO Evaluations (specification, status, timestamp,
|
|
|
|
checkouttime, evaltime)
|
2020-09-10 14:25:19 +02:00
|
|
|
VALUES (" spec-name "," (evaluation-status started) ","
|
2021-01-05 10:20:34 +01:00
|
|
|
now "," checkouttime "," evaltime ")
|
|
|
|
RETURNING id;"))
|
|
|
|
((id) (string->number id))))
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(new-checkouts (filter-map
|
|
|
|
(cut db-add-checkout spec-name eval-id <>)
|
|
|
|
checkouts)))
|
|
|
|
(if (null? new-checkouts)
|
2021-01-05 10:20:34 +01:00
|
|
|
(begin (exec-query db "ROLLBACK;")
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
#f)
|
2019-10-28 08:56:51 +01:00
|
|
|
(begin (db-add-event 'evaluation
|
|
|
|
(time-second (current-time time-utc))
|
|
|
|
`((#:evaluation . ,eval-id)
|
|
|
|
(#:specification . ,spec-name)
|
|
|
|
(#:in_progress . #t)))
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query db "COMMIT;")
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
eval-id)))))
|
|
|
|
|
2020-09-10 14:25:19 +02:00
|
|
|
(define (db-abort-pending-evaluations)
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "UPDATE Evaluations SET status =
|
2020-09-10 14:25:19 +02:00
|
|
|
" (evaluation-status aborted) " WHERE status = "
|
|
|
|
(evaluation-status started))))
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
|
2020-09-10 14:25:19 +02:00
|
|
|
(define (db-set-evaluation-status eval-id status)
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "UPDATE Evaluations SET status =
|
2020-09-10 14:25:19 +02:00
|
|
|
" status " WHERE id = " eval-id ";")))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
2020-09-06 13:03:08 +02:00
|
|
|
(define (db-set-evaluation-time eval-id)
|
|
|
|
(define now
|
|
|
|
(time-second (current-time time-utc)))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "UPDATE Evaluations SET evaltime = " now
|
|
|
|
"WHERE id = " eval-id ";")))
|
2017-08-26 10:42:40 +02:00
|
|
|
|
2018-01-23 17:37:08 +01:00
|
|
|
(define-enumeration build-status
|
2018-01-23 18:15:42 +01:00
|
|
|
;; Build status as expected by Hydra's API. Note: the negative values are
|
|
|
|
;; Cuirass' own extensions.
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(submitted -3)
|
2018-01-23 18:15:42 +01:00
|
|
|
(scheduled -2)
|
|
|
|
(started -1)
|
2018-01-23 17:37:08 +01:00
|
|
|
(succeeded 0)
|
|
|
|
(failed 1)
|
|
|
|
(failed-dependency 2)
|
|
|
|
(failed-other 3)
|
2018-01-23 23:16:32 +01:00
|
|
|
(canceled 4))
|
2018-01-23 17:37:08 +01:00
|
|
|
|
2018-09-02 09:45:48 +02:00
|
|
|
(define (db-add-output derivation output)
|
2021-01-05 10:20:34 +01:00
|
|
|
"Insert OUTPUT associated with DERIVATION."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(match output
|
|
|
|
((name . path)
|
|
|
|
(exec-query/bind db "\
|
2018-09-02 09:45:48 +02:00
|
|
|
INSERT INTO Outputs (derivation, name, path) VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
derivation ", " name ", " path ")
|
|
|
|
ON CONFLICT ON CONSTRAINT outputs_pkey DO NOTHING;")))))
|
2018-09-02 09:45:48 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-add-build build)
|
2018-09-02 09:45:48 +02:00
|
|
|
"Store BUILD in database the database only if one of its outputs is new.
|
|
|
|
Return #f otherwise. BUILD outputs are stored in the OUTPUTS table."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
INSERT INTO Builds (derivation, evaluation, job_name, system, nix_name, log,
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
status, priority, max_silent, timeout, timestamp, starttime, stoptime)
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
(assq-ref build #:derivation) ", "
|
|
|
|
(assq-ref build #:eval-id) ", "
|
|
|
|
(assq-ref build #:job-name) ", "
|
|
|
|
(assq-ref build #:system) ", "
|
|
|
|
(assq-ref build #:nix-name) ", "
|
|
|
|
(assq-ref build #:log) ", "
|
|
|
|
(or (assq-ref build #:status)
|
|
|
|
(build-status scheduled)) ", "
|
|
|
|
(or (assq-ref build #:priority) max-priority) ", "
|
|
|
|
(or (assq-ref build #:max-silent) 0) ", "
|
|
|
|
(or (assq-ref build #:timeout) 0) ", "
|
|
|
|
(or (assq-ref build #:timestamp) 0) ", "
|
|
|
|
(or (assq-ref build #:starttime) 0) ", "
|
|
|
|
(or (assq-ref build #:stoptime) 0) ")
|
|
|
|
ON CONFLICT ON CONSTRAINT builds_derivation_key DO NOTHING;"))
|
|
|
|
(let* ((derivation (assq-ref build #:derivation))
|
|
|
|
(outputs (assq-ref build #:outputs))
|
|
|
|
(new-outputs (filter-map (cut db-add-output derivation <>)
|
|
|
|
outputs)))
|
|
|
|
(db-add-event 'build
|
|
|
|
(assq-ref build #:timestamp)
|
|
|
|
`((#:derivation . ,derivation)
|
|
|
|
;; TODO Ideally this would use the value
|
|
|
|
;; from build, with a default of scheduled,
|
|
|
|
;; but it's hard to convert to the symbol,
|
|
|
|
;; so just hard code scheduled for now.
|
|
|
|
(#:event . scheduled)))
|
|
|
|
derivation))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-06-10 15:58:11 +02:00
|
|
|
(define (db-add-build-product product)
|
|
|
|
"Insert PRODUCT into BuildProducts table."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "\
|
|
|
|
INSERT INTO BuildProducts (build, type, file_size, checksum,
|
2020-06-10 15:58:11 +02:00
|
|
|
path) VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
(assq-ref product #:build) ", "
|
|
|
|
(assq-ref product #:type) ", "
|
|
|
|
(assq-ref product #:file-size) ", "
|
|
|
|
(assq-ref product #:checksum) ", "
|
|
|
|
(assq-ref product #:path) ");")))
|
|
|
|
|
|
|
|
(define (db-get-output path)
|
|
|
|
"Retrieve the OUTPUT for PATH."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(match (exec-query/bind db "SELECT derivation, name FROM Outputs
|
|
|
|
WHERE path =" path "
|
|
|
|
LIMIT 1;")
|
|
|
|
(() #f)
|
|
|
|
(((derivation name))
|
|
|
|
`((#:derivation . ,derivation)
|
|
|
|
(#:name . ,name))))))
|
|
|
|
|
|
|
|
(define (db-get-outputs derivation)
|
|
|
|
"Retrieve the OUTPUTS of the build identified by DERIVATION in the
|
|
|
|
database."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(let loop ((rows
|
|
|
|
(exec-query/bind db "SELECT name, path FROM Outputs
|
|
|
|
WHERE derivation =" derivation ";"))
|
|
|
|
(outputs '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse outputs))
|
|
|
|
(((name path)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `(,name . ((#:path . ,path)))
|
|
|
|
outputs)))))))
|
|
|
|
|
|
|
|
(define (db-get-time-since-previous-build job-name specification)
|
|
|
|
"Return the time difference in seconds between the current time and the
|
|
|
|
registration time of the last build for JOB-NAME and SPECIFICATION."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "
|
|
|
|
SELECT extract(epoch from now())::int - Builds.timestamp FROM Builds
|
|
|
|
INNER JOIN Evaluations on Builds.evaluation = Evaluations.id
|
|
|
|
WHERE job_name = " job-name "AND specification = " specification
|
|
|
|
"ORDER BY Builds.timestamp DESC LIMIT 1"))
|
|
|
|
((time)
|
|
|
|
(string->number time))
|
|
|
|
(else #f))))
|
2020-06-10 15:58:11 +02:00
|
|
|
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(define (db-register-builds jobs eval-id specification)
|
2020-09-29 14:34:14 +02:00
|
|
|
(define (new-outputs? outputs)
|
|
|
|
(let ((new-outputs
|
|
|
|
(filter-map (match-lambda
|
|
|
|
((name . path)
|
|
|
|
(let ((drv (db-get-output path)))
|
|
|
|
(and (not drv) path))))
|
|
|
|
outputs)))
|
|
|
|
(not (null? new-outputs))))
|
|
|
|
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(define (build-priority priority)
|
|
|
|
(let ((spec-priority (assq-ref specification #:priority)))
|
|
|
|
(+ (* spec-priority 10) priority)))
|
|
|
|
|
2020-09-29 14:34:14 +02:00
|
|
|
(define (register job)
|
2021-01-05 10:20:34 +01:00
|
|
|
(let* ((drv (assq-ref job #:derivation))
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(job-name (assq-ref job #:job-name))
|
|
|
|
(system (assq-ref job #:system))
|
|
|
|
(nix-name (assq-ref job #:nix-name))
|
|
|
|
(log (assq-ref job #:log))
|
|
|
|
(period (assq-ref job #:period))
|
|
|
|
(priority (or (assq-ref job #:priority) max-priority))
|
|
|
|
(max-silent (assq-ref job #:max-silent-time))
|
|
|
|
(timeout (assq-ref job #:timeout))
|
|
|
|
(outputs (assq-ref job #:outputs))
|
|
|
|
(cur-time (time-second (current-time time-utc))))
|
2020-09-29 14:34:14 +02:00
|
|
|
(and (new-outputs? outputs)
|
|
|
|
(let ((build `((#:derivation . ,drv)
|
|
|
|
(#:eval-id . ,eval-id)
|
|
|
|
(#:job-name . ,job-name)
|
|
|
|
(#:system . ,system)
|
|
|
|
(#:nix-name . ,nix-name)
|
|
|
|
|
|
|
|
;; XXX: We'd leave LOG to #f (i.e., NULL) but that
|
|
|
|
;; currently violates the non-NULL constraint.
|
|
|
|
(#:log . ,(or log ""))
|
|
|
|
|
|
|
|
(#:status . ,(build-status scheduled))
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(#:priority . ,(build-priority priority))
|
|
|
|
(#:max-silent . ,max-silent)
|
|
|
|
(#:timeout . ,timeout)
|
2020-09-29 14:34:14 +02:00
|
|
|
(#:outputs . ,outputs)
|
|
|
|
(#:timestamp . ,cur-time)
|
|
|
|
(#:starttime . 0)
|
|
|
|
(#:stoptime . 0))))
|
2020-11-24 17:54:31 +01:00
|
|
|
(if period
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(let* ((spec (assq-ref specification #:name))
|
2020-11-25 15:49:04 +01:00
|
|
|
(time
|
|
|
|
(db-get-time-since-previous-build job-name spec))
|
2020-11-24 17:54:31 +01:00
|
|
|
(add-build? (cond
|
|
|
|
((not time) #t)
|
|
|
|
((> time period) #t)
|
|
|
|
(else #f))))
|
|
|
|
(and add-build? (db-add-build build)))
|
|
|
|
(db-add-build build))))))
|
2020-09-29 14:34:14 +02:00
|
|
|
|
2020-10-14 13:49:03 +02:00
|
|
|
;; Use the database worker dedicated to write queries. We don't want this
|
|
|
|
;; query to be queued as it is already a quite large transaction by itself,
|
|
|
|
;; so pass the #:FORCE? option.
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
2020-10-01 18:18:33 +02:00
|
|
|
(log-message "Registering builds for evaluation ~a." eval-id)
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query db "BEGIN TRANSACTION;")
|
2020-10-08 12:02:40 +02:00
|
|
|
(let ((derivations (filter-map register jobs)))
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query db "COMMIT;")
|
2020-10-08 12:02:40 +02:00
|
|
|
derivations)))
|
2020-09-29 14:34:14 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define* (db-update-build-status! drv status #:key log-file)
|
|
|
|
"Update the database so that DRV's status is STATUS. This also updates the
|
2018-01-24 23:40:07 +01:00
|
|
|
'starttime' or 'stoptime' fields. If LOG-FILE is true, record it as the build
|
|
|
|
log file for DRV."
|
2018-01-23 18:15:42 +01:00
|
|
|
(define now
|
|
|
|
(time-second (current-time time-utc)))
|
|
|
|
|
2019-10-21 23:43:16 +02:00
|
|
|
(define status-names
|
|
|
|
`((,(build-status succeeded) . "succeeded")
|
|
|
|
(,(build-status failed) . "failed")
|
|
|
|
(,(build-status failed-dependency) . "failed (dependency)")
|
|
|
|
(,(build-status failed-other) . "failed (other)")
|
|
|
|
(,(build-status canceled) . "canceled")))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(if (= status (build-status started))
|
2019-10-21 23:43:16 +02:00
|
|
|
(begin
|
2021-01-05 10:20:34 +01:00
|
|
|
(if log-file
|
|
|
|
(exec-query/bind db "UPDATE Builds SET starttime=" now
|
|
|
|
",status=" status ",log=" log-file
|
|
|
|
"WHERE derivation=" drv ";")
|
|
|
|
(exec-query/bind db "UPDATE Builds SET starttime=" now
|
|
|
|
",status="
|
|
|
|
status "WHERE derivation=" drv ";"))
|
2019-10-21 23:43:16 +02:00
|
|
|
(db-add-event 'build
|
|
|
|
now
|
|
|
|
`((#:derivation . ,drv)
|
|
|
|
(#:event . started))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
|
|
|
;; Update only if we're switching to a different status; otherwise
|
|
|
|
;; leave things unchanged. This ensures that 'stoptime' remains valid
|
|
|
|
;; and doesn't change every time we mark DRV as 'succeeded' several
|
|
|
|
;; times in a row, for instance.
|
2019-10-21 23:43:16 +02:00
|
|
|
(begin
|
2021-01-05 10:20:34 +01:00
|
|
|
(let ((rows
|
|
|
|
(exec-query/bind db "UPDATE Builds SET stoptime=" now
|
|
|
|
", status=" status
|
|
|
|
"WHERE derivation=" drv
|
|
|
|
" AND status != " status ";")))
|
|
|
|
(when (positive? rows)
|
|
|
|
(db-add-event 'build
|
|
|
|
now
|
|
|
|
`((#:derivation . ,drv)
|
|
|
|
(#:event . ,(assq-ref status-names
|
|
|
|
status))))))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(define* (db-update-build-worker! drv worker)
|
|
|
|
"Update the database so that DRV's worker is WORKER."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(exec-query/bind db "UPDATE Builds SET worker=" worker
|
|
|
|
"WHERE derivation=" drv ";")))
|
2017-07-31 19:25:28 +02:00
|
|
|
|
2019-05-21 11:40:28 +02:00
|
|
|
(define (query->bind-arguments query-string)
|
|
|
|
"Return a list of keys to query strings by parsing QUERY-STRING."
|
2020-04-15 19:50:25 +02:00
|
|
|
(define status-values
|
|
|
|
`(("success" . ,(build-status succeeded))
|
|
|
|
("failed" . ,(build-status failed))
|
|
|
|
("failed-dependency" . ,(build-status failed-dependency))
|
|
|
|
("failed-other" . ,(build-status failed-other))
|
|
|
|
("canceled" . ,(build-status canceled))))
|
2021-01-05 10:20:34 +01:00
|
|
|
(let ((args (map
|
2019-06-20 23:31:34 +02:00
|
|
|
(lambda (token)
|
|
|
|
(match (string-split token #\:)
|
|
|
|
(("system" system)
|
2021-01-05 10:20:34 +01:00
|
|
|
`(#:system . ,system))
|
2019-06-20 23:31:34 +02:00
|
|
|
(("spec" spec)
|
2021-01-05 10:20:34 +01:00
|
|
|
`(#:spec . ,spec))
|
2020-04-15 19:50:25 +02:00
|
|
|
(("status" status)
|
2021-01-05 10:20:34 +01:00
|
|
|
`(#:status . ,(assoc-ref status-values status)))
|
2019-06-20 23:31:34 +02:00
|
|
|
((_ invalid) '()) ; ignore
|
|
|
|
((query)
|
2020-10-05 14:46:31 +02:00
|
|
|
;; Remove any '%' that could make the search too slow and
|
|
|
|
;; add one at the end of the query.
|
2021-01-05 10:20:34 +01:00
|
|
|
`(#:query . ,(string-append
|
|
|
|
(string-join
|
|
|
|
(string-split query #\%)
|
|
|
|
"")
|
|
|
|
"%")))))
|
2019-06-20 23:31:34 +02:00
|
|
|
(string-tokenize query-string))))
|
2019-05-21 11:40:28 +02:00
|
|
|
;; Normalize arguments
|
|
|
|
(fold (lambda (key acc)
|
2021-01-05 10:20:34 +01:00
|
|
|
(if (assq key acc)
|
2019-05-21 11:40:28 +02:00
|
|
|
acc
|
2021-01-05 10:20:34 +01:00
|
|
|
(cons (cons key #f) acc)))
|
2019-05-21 11:40:28 +02:00
|
|
|
args '(#:spec #:system))))
|
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define (db-get-build-products build-id)
|
|
|
|
"Return the build products associated to the given BUILD-ID."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(let loop ((rows (exec-query/bind db "
|
|
|
|
SELECT id, type, file_size, checksum, path from BuildProducts
|
|
|
|
WHERE build = " build-id))
|
|
|
|
(products '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse products))
|
|
|
|
(((id type file-size checksum path)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,(string->number id))
|
|
|
|
(#:type . ,type)
|
|
|
|
(#:file-size . ,(string->number file-size))
|
|
|
|
(#:checksum . ,checksum)
|
|
|
|
(#:path . ,path))
|
|
|
|
products)))))))
|
|
|
|
|
2019-05-17 22:33:27 +02:00
|
|
|
(define (db-get-builds-by-search filters)
|
|
|
|
"Retrieve all builds in the database which are matched by given FILTERS.
|
|
|
|
FILTERS is an assoc list whose possible keys are the symbols query,
|
|
|
|
border-low-id, border-high-id, and nr."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let* ((query (format #f "SELECT Builds.id, Builds.timestamp,
|
2020-10-05 14:46:31 +02:00
|
|
|
Builds.starttime,Builds.stoptime, Builds.log, Builds.status,
|
|
|
|
Builds.job_name, Builds.system, Builds.nix_name, Specifications.name
|
2019-05-17 22:33:27 +02:00
|
|
|
FROM Builds
|
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
|
|
|
WHERE (Builds.nix_name LIKE :query)
|
2021-01-05 10:20:34 +01:00
|
|
|
AND ((Builds.status = :status) OR :status IS NULL)
|
|
|
|
AND ((Specifications.name = :spec) OR :spec IS NULL)
|
|
|
|
AND ((Builds.system = :system) OR :system IS NULL)
|
|
|
|
AND ((:borderlowid < Builds.id) OR :borderlowid IS NULL)
|
|
|
|
AND ((:borderhighid > Builds.id) OR :borderhighid IS NULL)
|
2019-05-17 22:33:27 +02:00
|
|
|
ORDER BY
|
2021-01-05 10:20:34 +01:00
|
|
|
CASE WHEN :borderlowid IS NULL THEN Builds.id
|
|
|
|
ELSE -Builds.id
|
2019-05-17 22:33:27 +02:00
|
|
|
END DESC
|
2020-10-05 14:46:31 +02:00
|
|
|
LIMIT :nr;"))
|
2021-01-05 10:20:34 +01:00
|
|
|
(builds
|
|
|
|
(exec-query/bind-params
|
|
|
|
db
|
|
|
|
query
|
|
|
|
`((#:borderlowid . ,(assq-ref filters 'border-low-id))
|
|
|
|
(#:borderhighid . ,(assq-ref filters 'border-high-id))
|
|
|
|
(#:nr . ,(match (assq-ref filters 'nr)
|
|
|
|
(#f -1)
|
|
|
|
(x x)))
|
|
|
|
,@(query->bind-arguments (assq-ref filters 'query))))))
|
|
|
|
(let loop ((builds builds)
|
|
|
|
(result '()))
|
|
|
|
(match builds
|
|
|
|
(() result)
|
|
|
|
(((id timestamp starttime stoptime log status job-name
|
|
|
|
system nix-name specification)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,(string->number id))
|
|
|
|
(#:timestamp . ,(string->number timestamp))
|
|
|
|
(#:starttime . ,(string->number starttime))
|
|
|
|
(#:stoptime . ,(string->number stoptime))
|
|
|
|
(#:log . ,log)
|
|
|
|
(#:status . ,(string->number status))
|
|
|
|
(#:job-name . ,job-name)
|
|
|
|
(#:system . ,system)
|
|
|
|
(#:nix-name . ,nix-name)
|
|
|
|
(#:specification . ,specification)
|
|
|
|
(#:buildproducts . ,(db-get-build-products id)))
|
|
|
|
result))))))))
|
2019-05-17 22:33:27 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-builds filters)
|
|
|
|
"Retrieve all builds in the database which are matched by given FILTERS.
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
FILTERS is an assoc list whose possible keys are 'derivation | 'id | 'jobset |
|
|
|
|
'job | 'system | 'nr | 'order | 'status | 'evaluation."
|
2020-05-25 16:12:07 +02:00
|
|
|
|
2020-10-07 11:49:30 +02:00
|
|
|
;; XXX: Make sure that all filters are covered by an index.
|
2020-05-25 16:12:07 +02:00
|
|
|
(define (filters->order filters)
|
2020-12-02 17:16:08 +01:00
|
|
|
(lambda (inner)
|
|
|
|
(match (assq 'order filters)
|
2021-01-05 10:20:34 +01:00
|
|
|
(('order . 'build-id) "Builds.id ASC")
|
2020-12-02 17:16:08 +01:00
|
|
|
(('order . 'finish-time) "stoptime DESC")
|
|
|
|
(('order . 'finish-time+build-id)
|
|
|
|
(if inner
|
2021-01-05 10:20:34 +01:00
|
|
|
"CASE WHEN CAST(:borderlowid AS integer) IS NULL THEN
|
2020-12-02 17:16:08 +01:00
|
|
|
stoptime ELSE -stoptime END DESC,
|
2021-01-05 10:20:34 +01:00
|
|
|
CASE WHEN CAST(:borderlowid AS integer) IS NULL THEN
|
|
|
|
Builds.id ELSE -Builds.id END DESC"
|
|
|
|
"stoptime DESC, Builds.id DESC"))
|
2020-12-02 17:16:08 +01:00
|
|
|
;; With this order, builds in 'running' state (-1) appear
|
|
|
|
;; before those in 'scheduled' state (-2).
|
|
|
|
(('order . 'status+submission-time)
|
2021-01-05 10:20:34 +01:00
|
|
|
"Builds.status DESC, Builds.timestamp DESC, Builds.id ASC")
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(('order . 'priority+timestamp)
|
2021-01-05 10:20:34 +01:00
|
|
|
"Builds.priority ASC, Builds.timestamp DESC")
|
|
|
|
(_ "Builds.id DESC"))))
|
2020-05-25 16:12:07 +02:00
|
|
|
|
2020-10-07 11:49:30 +02:00
|
|
|
;; XXX: Make sure that all filters are covered by an index.
|
2020-05-25 16:12:07 +02:00
|
|
|
(define (where-conditions filters)
|
|
|
|
(define filter-name->sql
|
|
|
|
`((id . "Builds.id = :id")
|
|
|
|
(jobset . "Specifications.name = :jobset")
|
|
|
|
(derivation . "Builds.derivation = :derivation")
|
|
|
|
(job . "Builds.job_name = :job")
|
|
|
|
(system . "Builds.system = :system")
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
(worker . "Builds.worker = :worker")
|
2020-05-25 16:12:07 +02:00
|
|
|
(evaluation . "Builds.evaluation = :evaluation")
|
|
|
|
(status . ,(match (assq-ref filters 'status)
|
|
|
|
(#f #f)
|
|
|
|
('done "Builds.status >= 0")
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
('scheduled "Builds.status = -2")
|
2020-07-02 21:07:21 +02:00
|
|
|
('started "Builds.status = -1")
|
2020-05-25 16:12:07 +02:00
|
|
|
('pending "Builds.status < 0")
|
|
|
|
('succeeded "Builds.status = 0")
|
|
|
|
('failed "Builds.status > 0")))
|
2020-12-02 17:16:08 +01:00
|
|
|
(border-low-time
|
2021-01-05 10:20:34 +01:00
|
|
|
. "(((:borderlowtime, :borderlowid) < (Builds.stoptime, Builds.id))
|
|
|
|
OR :borderlowtime IS NULL OR :borderlowid IS NULL)")
|
2020-12-02 17:16:08 +01:00
|
|
|
(border-high-time
|
2021-01-05 10:20:34 +01:00
|
|
|
. "(((:borderhightime, :borderhighid) > (Builds.stoptime, Builds.id))
|
|
|
|
OR :borderhightime IS NULL OR :borderhighid IS NULL)")))
|
2020-05-25 16:12:07 +02:00
|
|
|
|
|
|
|
(filter
|
|
|
|
string?
|
|
|
|
(fold
|
|
|
|
(lambda (filter-name where-condition-parts)
|
|
|
|
(if (assq-ref filters filter-name)
|
|
|
|
(cons (assq-ref filter-name->sql filter-name)
|
|
|
|
where-condition-parts)
|
|
|
|
where-condition-parts))
|
|
|
|
'()
|
|
|
|
(map car filters))))
|
|
|
|
|
2020-10-07 11:49:30 +02:00
|
|
|
(define (format-outputs names paths)
|
|
|
|
(map (lambda (name path)
|
|
|
|
`(,name . ((#:path . ,path))))
|
|
|
|
(string-split names #\,)
|
|
|
|
(string-split paths #\,)))
|
|
|
|
|
|
|
|
(define (format-build-products ids types file-sizes checksums paths)
|
|
|
|
(define (split list)
|
|
|
|
(if list
|
|
|
|
(string-split list #\,)
|
|
|
|
'()))
|
|
|
|
|
|
|
|
(map (lambda (id type file-size checksum path)
|
|
|
|
`((#:id . ,(string->number id))
|
|
|
|
(#:type . ,type)
|
|
|
|
(#:file-size . ,(string->number file-size))
|
|
|
|
(#:checksum . ,checksum)
|
|
|
|
(#:path . ,path)))
|
|
|
|
(split ids)
|
|
|
|
(split types)
|
|
|
|
(split file-sizes)
|
|
|
|
(split checksums)
|
|
|
|
(split paths)))
|
|
|
|
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let* ((order (filters->order filters))
|
2020-05-25 16:12:07 +02:00
|
|
|
(where (match (where-conditions filters)
|
|
|
|
(() "")
|
|
|
|
((condition)
|
|
|
|
(string-append "WHERE " condition "\n"))
|
|
|
|
((first-condition rest ...)
|
|
|
|
(string-append "WHERE " first-condition "\n AND "
|
|
|
|
(string-join rest " AND ")))))
|
2021-01-05 10:20:34 +01:00
|
|
|
(query
|
|
|
|
(format #f " SELECT Builds.derivation, Builds.id, Builds.timestamp,
|
|
|
|
Builds.starttime, Builds.stoptime, Builds.log, Builds.status, Builds.priority,
|
|
|
|
Builds.max_silent, Builds.timeout, Builds.job_name, Builds.system,
|
|
|
|
Builds.nix_name, Builds.evaluation, agg.name, agg.outputs_name,
|
|
|
|
agg.outputs_path,agg.bp_build, agg.bp_type, agg.bp_file_size,
|
|
|
|
agg.bp_checksum, agg.bp_path
|
|
|
|
FROM
|
|
|
|
(SELECT B.id, B.derivation, B.name,
|
|
|
|
string_agg(Outputs.name, ',') AS outputs_name,
|
|
|
|
string_agg(Outputs.path, ',') AS outputs_path,
|
2021-01-17 11:53:47 +01:00
|
|
|
string_agg(cast(BP.id AS text), ',') AS bp_build,
|
2021-01-05 10:20:34 +01:00
|
|
|
string_agg(BP.type, ',') AS bp_type,
|
|
|
|
string_agg(cast(BP.file_size AS text), ',') AS bp_file_size,
|
|
|
|
string_agg(BP.checksum, ',') AS bp_checksum,
|
|
|
|
string_agg(BP.path, ',') AS bp_path FROM
|
|
|
|
(SELECT Builds.id, Builds.derivation, Specifications.name FROM Builds
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
2020-05-25 16:12:07 +02:00
|
|
|
~a
|
|
|
|
ORDER BY ~a
|
2021-01-05 10:20:34 +01:00
|
|
|
LIMIT :nr) B
|
|
|
|
INNER JOIN Outputs ON Outputs.derivation = B.derivation
|
|
|
|
LEFT JOIN BuildProducts as BP ON BP.build = B.id
|
|
|
|
GROUP BY B.derivation, B.id, B.name) agg
|
|
|
|
JOIN Builds on agg.id = Builds.id
|
2020-10-07 11:49:30 +02:00
|
|
|
ORDER BY ~a;"
|
2020-12-02 17:16:08 +01:00
|
|
|
where (order #t) (order #f)))
|
2021-01-05 10:20:34 +01:00
|
|
|
(params
|
|
|
|
(map (match-lambda
|
|
|
|
((name . value)
|
2021-01-17 12:05:31 +01:00
|
|
|
(cons (symbol->keyword
|
|
|
|
(or (assq-ref
|
|
|
|
'((border-low-time . borderlowtime)
|
|
|
|
(border-high-time . borderhightime)
|
|
|
|
(border-low-id . borderlowid)
|
|
|
|
(border-high-id . borderhighid))
|
|
|
|
name)
|
|
|
|
name))
|
|
|
|
(match name
|
|
|
|
('nr (or value -1))
|
|
|
|
('order #f) ; Doesn't need binding.
|
|
|
|
('status #f) ; Doesn't need binding.
|
|
|
|
(else value)))))
|
2021-01-05 10:20:34 +01:00
|
|
|
filters))
|
|
|
|
(builds (exec-query/bind-params db query params)))
|
|
|
|
(let loop ((builds builds)
|
|
|
|
(result '()))
|
|
|
|
(match builds
|
|
|
|
(() (reverse result))
|
|
|
|
(((derivation id timestamp starttime stoptime log status
|
|
|
|
priority max-silent timeout job-name
|
|
|
|
system nix-name eval-id specification
|
|
|
|
outputs-name outputs-path
|
|
|
|
products-id products-type products-file-size
|
|
|
|
products-checksum products-path)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:derivation . ,derivation)
|
|
|
|
(#:id . ,(string->number id))
|
|
|
|
(#:timestamp . ,(string->number timestamp))
|
|
|
|
(#:starttime . ,(string->number starttime))
|
|
|
|
(#:stoptime . ,(string->number stoptime))
|
|
|
|
(#:log . ,log)
|
|
|
|
(#:status . ,(string->number status))
|
|
|
|
(#:priority . ,(string->number priority))
|
|
|
|
(#:max-silent . ,(string->number max-silent))
|
|
|
|
(#:timeout . ,(string->number timeout))
|
|
|
|
(#:job-name . ,job-name)
|
|
|
|
(#:system . ,system)
|
|
|
|
(#:nix-name . ,nix-name)
|
|
|
|
(#:eval-id . ,(string->number eval-id))
|
|
|
|
(#:specification . ,specification)
|
|
|
|
(#:outputs . ,(format-outputs outputs-name
|
|
|
|
outputs-path))
|
|
|
|
(#:buildproducts .
|
|
|
|
,(format-build-products products-id
|
|
|
|
products-type
|
|
|
|
products-file-size
|
|
|
|
products-checksum
|
|
|
|
products-path)))
|
|
|
|
result))))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
|
|
|
(define (db-get-build derivation-or-id)
|
|
|
|
"Retrieve a build in the database which corresponds to DERIVATION-OR-ID."
|
2021-01-05 10:20:34 +01:00
|
|
|
(let ((key (if (number? derivation-or-id) 'id 'derivation)))
|
|
|
|
(expect-one-row (db-get-builds `((,key . ,derivation-or-id))))))
|
2019-10-21 23:43:16 +02:00
|
|
|
|
|
|
|
(define (db-get-events filters)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let* ((query "\
|
2019-10-21 23:43:16 +02:00
|
|
|
SELECT Events.id,
|
|
|
|
Events.type,
|
|
|
|
Events.timestamp,
|
|
|
|
Events.event_json
|
|
|
|
FROM Events
|
2021-01-05 10:20:34 +01:00
|
|
|
WHERE (:type = Events.type OR :type IS NULL)
|
|
|
|
AND (((:borderlowtime, :borderlowid) <
|
|
|
|
(Events.timestamp, Events.id)) OR
|
|
|
|
:borderlowtime IS NULL OR
|
|
|
|
:borderlowid IS NULL)
|
|
|
|
AND (((:borderhightime, :borderhighid) >
|
|
|
|
(Events.timestamp, Events.id)) OR
|
|
|
|
:borderhightime IS NULL OR
|
|
|
|
:borderhighid IS NULL)
|
2019-10-21 23:43:16 +02:00
|
|
|
ORDER BY Events.id ASC
|
|
|
|
LIMIT :nr;")
|
2021-01-05 10:20:34 +01:00
|
|
|
(params `((#:type . ,(and=> (assq-ref filters 'type)
|
|
|
|
symbol->string))
|
|
|
|
(#:nr . ,(match (assq-ref filters 'nr)
|
|
|
|
(#f -1)
|
|
|
|
(x x)))))
|
|
|
|
(events (exec-query/bind-params db query params)))
|
|
|
|
(let loop ((events events)
|
|
|
|
(result '()))
|
|
|
|
(match events
|
|
|
|
(() (reverse result))
|
|
|
|
(((id type timestamp event_json)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,(string->number id))
|
|
|
|
(#:type . ,(string->symbol type))
|
|
|
|
(#:timestamp . ,(string->number timestamp))
|
|
|
|
(#:event_json . ,event_json))
|
|
|
|
result))))))))
|
2019-10-21 23:43:16 +02:00
|
|
|
|
|
|
|
(define (db-delete-events-with-ids-<=-to id)
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "DELETE FROM Events WHERE id <= " id ";")))
|
2019-10-21 23:43:16 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-pending-derivations)
|
2018-04-05 22:15:20 +02:00
|
|
|
"Return the list of derivation file names corresponding to pending builds in
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
the database. The returned list is guaranteed to not have any duplicates."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(map (match-lambda ((drv) drv))
|
|
|
|
(exec-query db "
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
SELECT derivation FROM Builds WHERE Builds.status < 0;"))))
|
|
|
|
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(define (db-get-checkouts eval-id)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let loop ((rows (exec-query/bind
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
db "SELECT revision, input, directory FROM Checkouts
|
|
|
|
WHERE evaluation =" eval-id ";"))
|
|
|
|
(checkouts '()))
|
|
|
|
(match rows
|
2021-01-05 10:20:34 +01:00
|
|
|
(() (reverse checkouts))
|
|
|
|
(((revision input directory)
|
|
|
|
. rest)
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(loop rest
|
|
|
|
(cons `((#:commit . ,revision)
|
|
|
|
(#:input . ,input)
|
|
|
|
(#:directory . ,directory))
|
|
|
|
checkouts)))))))
|
2018-04-19 11:17:42 +02:00
|
|
|
|
2021-01-05 10:20:34 +01:00
|
|
|
(define (parse-evaluation evaluation)
|
|
|
|
(match evaluation
|
|
|
|
((id specification status timestamp checkouttime evaltime)
|
|
|
|
`((#:id . ,(string->number id))
|
|
|
|
(#:specification . ,specification)
|
|
|
|
(#:status . ,(string->number status))
|
|
|
|
(#:timestamp . ,(string->number timestamp))
|
|
|
|
(#:checkouttime . ,(string->number checkouttime))
|
|
|
|
(#:evaltime . ,(string->number evaltime))
|
|
|
|
(#:checkouts . ,(db-get-checkouts id))))))
|
|
|
|
|
2020-12-07 15:34:29 +01:00
|
|
|
(define (db-get-evaluation id)
|
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(match (exec-query/bind db "SELECT id, specification, status,
|
2020-12-07 15:34:29 +01:00
|
|
|
timestamp, checkouttime, evaltime
|
|
|
|
FROM Evaluations WHERE id = " id)
|
|
|
|
(() #f)
|
2021-01-05 10:20:34 +01:00
|
|
|
((evaluation)
|
|
|
|
(parse-evaluation evaluation)))))
|
2020-12-07 15:34:29 +01:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-evaluations limit)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let loop ((rows (exec-query/bind db "SELECT id, specification, status,
|
2020-09-06 14:23:01 +02:00
|
|
|
timestamp, checkouttime, evaltime
|
2018-04-19 11:17:42 +02:00
|
|
|
FROM Evaluations ORDER BY id DESC LIMIT " limit ";"))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(evaluations '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse evaluations))
|
2021-01-05 10:20:34 +01:00
|
|
|
((evaluation . rest)
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(loop rest
|
2021-01-05 10:20:34 +01:00
|
|
|
(cons (parse-evaluation evaluation) evaluations)))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
|
|
|
(define (db-get-evaluations-build-summary spec limit border-low border-high)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let ((query "
|
|
|
|
SELECT E.id, E.status,
|
|
|
|
SUM(CASE WHEN B.status = 0 THEN 1 ELSE 0 END) as succeeded,
|
|
|
|
SUM(CASE WHEN B.status > 0 THEN 1 ELSE 0 END) as failed,
|
|
|
|
SUM(CASE WHEN B.status < 0 THEN 1 ELSE 0 END) as scheduled FROM
|
2020-09-28 17:47:19 +02:00
|
|
|
(SELECT id, status FROM Evaluations
|
2021-01-05 10:20:34 +01:00
|
|
|
WHERE specification=:spec
|
|
|
|
AND (id > :borderlow OR :borderlow IS NULL)
|
|
|
|
AND (id < :borderhigh OR :borderhigh IS NULL)
|
|
|
|
ORDER BY CASE WHEN :borderlow IS NULL THEN id ELSE -id END DESC
|
|
|
|
LIMIT :limit) E
|
2020-09-28 17:47:19 +02:00
|
|
|
LEFT JOIN Builds as B
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
ON B.evaluation=E.id
|
2021-01-05 10:20:34 +01:00
|
|
|
GROUP BY E.id, E.status
|
|
|
|
ORDER BY E.id DESC;")
|
|
|
|
(params `((#:spec . ,spec)
|
|
|
|
(#:limit . ,limit)
|
|
|
|
(#:borderlow . ,border-low)
|
|
|
|
(#:borderhigh . ,border-high))))
|
|
|
|
(let loop ((rows (exec-query/bind-params db query params))
|
|
|
|
(evaluations '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse evaluations))
|
|
|
|
(((id status succeeded failed scheduled) . rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,(string->number id))
|
|
|
|
(#:status . ,(string->number status))
|
|
|
|
(#:checkouts . ,(db-get-checkouts id))
|
|
|
|
(#:succeeded . ,(or (string->number succeeded) 0))
|
|
|
|
(#:failed . ,(or (string->number failed) 0))
|
|
|
|
(#:scheduled . ,(or (string->number scheduled) 0)))
|
|
|
|
evaluations))))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
|
|
|
(define (db-get-evaluations-id-min spec)
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
"Return the min id of evaluations for the given specification SPEC."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
SELECT MIN(id) FROM Evaluations
|
2021-01-05 10:20:34 +01:00
|
|
|
WHERE specification=" spec))
|
|
|
|
((min) (and min (string->number min))))))
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-evaluations-id-max spec)
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
"Return the max id of evaluations for the given specification SPEC."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
SELECT MAX(id) FROM Evaluations
|
2021-01-05 10:20:34 +01:00
|
|
|
WHERE specification=" spec))
|
|
|
|
((max) (and max (string->number max))))))
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
|
2019-06-19 10:21:33 +02:00
|
|
|
(define (db-get-evaluation-summary id)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "
|
|
|
|
SELECT Evaluations.id, Evaluations.status, Evaluations.timestamp,
|
|
|
|
Evaluations.checkouttime, Evaluations.evaltime,
|
|
|
|
SUM(CASE WHEN B.status > -100 THEN 1 ELSE 0 END) as total,
|
|
|
|
SUM(CASE WHEN B.status = 0 THEN 1 ELSE 0 END) as succeeded,
|
|
|
|
SUM(CASE WHEN B.status > 0 THEN 1 ELSE 0 END) as failed,
|
|
|
|
SUM(CASE WHEN B.status < 0 THEN 1 ELSE 0 END) as scheduled
|
|
|
|
FROM Evaluations
|
2020-09-28 17:47:19 +02:00
|
|
|
LEFT JOIN Builds as B
|
2021-01-05 10:20:34 +01:00
|
|
|
ON B.evaluation = Evaluations.id
|
|
|
|
WHERE Evaluations.id = " id
|
|
|
|
"GROUP BY Evaluations.id
|
|
|
|
ORDER BY Evaluations.id ASC;"))
|
|
|
|
((id status timestamp checkouttime evaltime
|
|
|
|
total succeeded failed scheduled)
|
|
|
|
`((#:id . ,(string->number id))
|
|
|
|
(#:status . ,(string->number status))
|
|
|
|
(#:total . ,(or (string->number total) 0))
|
|
|
|
(#:timestamp . ,(string->number timestamp))
|
|
|
|
(#:checkouttime . ,(string->number checkouttime))
|
|
|
|
(#:evaltime . ,(string->number evaltime))
|
|
|
|
(#:succeeded . ,(or (string->number succeeded) 0))
|
|
|
|
(#:failed . ,(or (string->number failed) 0))
|
|
|
|
(#:scheduled . ,(or (string->number scheduled) 0))))
|
|
|
|
(else #f))))
|
|
|
|
|
|
|
|
(define (db-get-builds-query-min filters)
|
2019-05-17 22:33:27 +02:00
|
|
|
"Return the smallest build row identifier matching QUERY."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let* ((query "SELECT MIN(Builds.id) FROM Builds
|
2019-05-21 11:40:28 +02:00
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
|
|
|
WHERE (Builds.nix_name LIKE :query)
|
2021-01-05 10:20:34 +01:00
|
|
|
AND (Builds.status = :status OR :status IS NULL)
|
|
|
|
AND (Specifications.name = :spec OR :spec IS NULL)
|
|
|
|
AND (Builds.system = :system OR :system IS NULL);")
|
|
|
|
(params (query->bind-arguments filters)))
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind-params db query params))
|
|
|
|
((min) (and min
|
|
|
|
(list (string->number min))))))))
|
|
|
|
|
|
|
|
(define (db-get-builds-query-max filters)
|
2019-05-17 22:33:27 +02:00
|
|
|
"Return the largest build row identifier matching QUERY."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let* ((query "SELECT MAX(Builds.id) FROM Builds
|
2019-05-21 11:40:28 +02:00
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
|
|
|
WHERE (Builds.nix_name LIKE :query)
|
2021-01-05 10:20:34 +01:00
|
|
|
AND (Builds.status = :status OR :status IS NULL)
|
|
|
|
AND (Specifications.name = :spec OR :spec IS NULL)
|
|
|
|
AND (Builds.system = :system OR :system IS NULL);")
|
|
|
|
(params (query->bind-arguments filters)))
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind-params db query params))
|
|
|
|
((max) (and max
|
|
|
|
(list (string->number max))))))))
|
2019-05-17 22:33:27 +02:00
|
|
|
|
2018-08-05 16:55:16 +02:00
|
|
|
(define (db-get-builds-min eval status)
|
|
|
|
"Return the min build (stoptime, rowid) pair for the given evaluation EVAL
|
|
|
|
and STATUS."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let ((query "SELECT stoptime, id FROM Builds
|
|
|
|
WHERE evaluation = :eval AND
|
|
|
|
((:status = 'pending' AND Builds.status < 0) OR
|
|
|
|
(:status = 'succeeded' AND Builds.status = 0) OR
|
|
|
|
(:status = 'failed' AND Builds.status > 0) OR
|
|
|
|
:status IS NULL)
|
|
|
|
ORDER BY stoptime ASC, id ASC
|
|
|
|
LIMIT 1")
|
|
|
|
(params `((#:eval . ,eval)
|
|
|
|
(#:status . ,status))))
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind-params db query params))
|
|
|
|
((stoptime id) (list (string->number stoptime)
|
|
|
|
(string->number id)))
|
|
|
|
(else #f)))))
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
|
2018-08-05 16:55:16 +02:00
|
|
|
(define (db-get-builds-max eval status)
|
|
|
|
"Return the max build (stoptime, rowid) pair for the given evaluation EVAL
|
|
|
|
and STATUS."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let ((query "SELECT stoptime, id FROM Builds
|
|
|
|
WHERE evaluation = :eval AND
|
|
|
|
((:status = 'pending' AND Builds.status < 0) OR
|
|
|
|
(:status = 'succeeded' AND Builds.status = 0) OR
|
|
|
|
(:status = 'failed' AND Builds.status > 0) OR
|
|
|
|
:status IS NULL)
|
|
|
|
ORDER BY stoptime DESC, id DESC
|
|
|
|
LIMIT 1")
|
|
|
|
(params `((#:eval . ,eval)
|
|
|
|
(#:status . ,status))))
|
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind-params db query params))
|
|
|
|
((stoptime id) (list (string->number stoptime)
|
|
|
|
(string->number id)))
|
|
|
|
(else #f)))))
|
2018-08-05 21:25:37 +02:00
|
|
|
|
|
|
|
(define (db-get-evaluation-specification eval)
|
|
|
|
"Return specification of evaluation with id EVAL."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "
|
2018-08-05 21:25:37 +02:00
|
|
|
SELECT specification FROM Evaluations
|
2021-01-05 10:20:34 +01:00
|
|
|
WHERE id = " eval))
|
|
|
|
((spec) spec)
|
|
|
|
(else #f))))
|
2020-06-10 15:58:11 +02:00
|
|
|
|
|
|
|
(define (db-get-build-product-path id)
|
|
|
|
"Return the build product with the given ID."
|
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(match (expect-one-row
|
|
|
|
(exec-query/bind db "
|
2020-06-10 15:58:11 +02:00
|
|
|
SELECT path FROM BuildProducts
|
2021-01-05 10:20:34 +01:00
|
|
|
WHERE id = " id))
|
|
|
|
((path) path)
|
|
|
|
(else #f))))
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
|
|
|
|
(define (db-add-worker worker)
|
|
|
|
"Insert WORKER into Worker table."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query/bind db "\
|
|
|
|
INSERT INTO Workers (name, address, systems, last_seen)
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
VALUES ("
|
2021-01-05 10:20:34 +01:00
|
|
|
(worker-name worker) ", "
|
|
|
|
(worker-address worker) ", "
|
|
|
|
(string-join (worker-systems worker) ",") ", "
|
|
|
|
(worker-last-seen worker) ");")))
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
|
|
|
|
(define (db-get-workers)
|
|
|
|
"Return the workers in Workers table."
|
|
|
|
(with-db-worker-thread db
|
2021-01-05 10:20:34 +01:00
|
|
|
(let loop ((rows (exec-query db "
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
SELECT name, address, systems, last_seen from Workers"))
|
|
|
|
(workers '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse workers))
|
2021-01-05 10:20:34 +01:00
|
|
|
(((name address systems last-seen)
|
Add remote build support.
* src/cuirass/remote.scm: New file.
* src/cuirass/remote-server.scm: New file.
* src/cuirass/remote-worker.scm: New file.
* bin/remote-server.in: New file.
* bin/remote-worker.in: New file.
* Makefile.am (bin_SCRIPTS): Add new binaries,
(dist_pkgmodule_DATA): add new files,
(EXTRA_DIST): add new binaries,
(bin/remote-server, bin/remote-worker): new targets.
* .gitignore: Add new binaries.
* bin/cuirass.in (%options): Add "--build-remote" option,
(show-help): document it,
(main): honor it.
* src/cuirass/base.scm (with-build-offload-thread): New macro,
(%build-remote?, %build-offload-channel): new parameters,
(make-build-offload-thread): new procedure,
(build-derivations/offload): new procedure,
(restart-builds): use it to offload builds when %build-remote? is set,
(build-packages): ditto.
2020-12-02 11:13:33 +01:00
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons (worker
|
|
|
|
(name name)
|
|
|
|
(address address)
|
|
|
|
(systems (string-split systems #\,))
|
|
|
|
(last-seen last-seen))
|
|
|
|
workers)))))))
|
|
|
|
|
|
|
|
(define (db-clear-workers)
|
|
|
|
"Remove all workers from Workers table."
|
2021-01-05 10:20:34 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query db "DELETE FROM Workers;")))
|
|
|
|
|
|
|
|
(define (db-clear-build-queue)
|
|
|
|
"Reset the status of builds in the database that are marked as \"started\"."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(exec-query db "UPDATE Builds SET status = -2 WHERE status < 0;")))
|
|
|
|
|
|
|
|
;;; Local Variables:
|
|
|
|
;;; eval: (put 'with-db-worker-thread 'scheme-indent-function 1)
|
|
|
|
;;; End:
|