2016-07-16 18:16:39 +02:00
|
|
|
;;; database.scm -- store evaluation and build results
|
2017-01-09 01:29:48 +01:00
|
|
|
;;; Copyright © 2016, 2017 Mathieu Lirzin <mthl@gnu.org>
|
2020-06-10 15:58:11 +02:00
|
|
|
;;; Copyright © 2017, 2020 Mathieu Othacehe <m.othacehe@gmail.com>
|
2020-02-23 23:25:23 +01:00
|
|
|
;;; Copyright © 2018, 2020 Ludovic Courtès <ludo@gnu.org>
|
2018-07-07 00:31:14 +02:00
|
|
|
;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
;;; Copyright © 2018 Tatiana Sholokhova <tanja201396@gmail.com>
|
2020-04-15 19:50:25 +02:00
|
|
|
;;; Copyright © 2019, 2020 Ricardo Wurmus <rekado@elephly.net>
|
2016-06-26 15:13:31 +02:00
|
|
|
;;;
|
|
|
|
;;; This file is part of Cuirass.
|
|
|
|
;;;
|
2016-07-23 12:10:04 +02:00
|
|
|
;;; Cuirass is free software: you can redistribute it and/or modify
|
|
|
|
;;; it under the terms of the GNU General Public License as published by
|
|
|
|
;;; the Free Software Foundation, either version 3 of the License, or
|
|
|
|
;;; (at your option) any later version.
|
2016-06-26 15:13:31 +02:00
|
|
|
;;;
|
2016-07-23 12:10:04 +02:00
|
|
|
;;; Cuirass is distributed in the hope that it will be useful,
|
|
|
|
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
|
2016-06-26 15:13:31 +02:00
|
|
|
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
;;; GNU General Public License for more details.
|
|
|
|
;;;
|
|
|
|
;;; You should have received a copy of the GNU General Public License
|
|
|
|
;;; along with Cuirass. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
(define-module (cuirass database)
|
2020-08-01 11:56:43 +02:00
|
|
|
#:use-module (cuirass logging)
|
2016-06-26 18:40:31 +02:00
|
|
|
#:use-module (cuirass config)
|
2016-07-23 22:05:50 +02:00
|
|
|
#:use-module (cuirass utils)
|
|
|
|
#:use-module (ice-9 match)
|
2018-01-24 23:40:07 +01:00
|
|
|
#:use-module (ice-9 format)
|
2018-07-07 00:31:14 +02:00
|
|
|
#:use-module (ice-9 ftw)
|
2016-07-16 18:16:39 +02:00
|
|
|
#:use-module (ice-9 rdelim)
|
2018-07-07 00:31:14 +02:00
|
|
|
#:use-module (ice-9 regex)
|
2020-04-17 14:23:18 +02:00
|
|
|
#:use-module (ice-9 threads)
|
2017-07-31 19:25:28 +02:00
|
|
|
#:use-module (srfi srfi-1)
|
2018-01-23 18:15:42 +01:00
|
|
|
#:use-module (srfi srfi-19)
|
2018-02-19 16:30:07 +01:00
|
|
|
#:use-module (srfi srfi-26)
|
2020-09-22 15:17:31 +02:00
|
|
|
#:use-module (system foreign)
|
|
|
|
#:use-module (rnrs bytevectors)
|
2016-06-26 15:13:31 +02:00
|
|
|
#:use-module (sqlite3)
|
2016-06-26 18:40:31 +02:00
|
|
|
#:export (;; Procedures.
|
|
|
|
db-init
|
2016-06-26 15:13:31 +02:00
|
|
|
db-open
|
|
|
|
db-close
|
2020-05-25 12:08:20 +02:00
|
|
|
db-optimize
|
2020-09-22 15:17:31 +02:00
|
|
|
db-log-queries
|
2016-07-23 17:00:38 +02:00
|
|
|
db-add-specification
|
2019-10-30 09:19:28 +01:00
|
|
|
db-remove-specification
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
db-get-specification
|
2016-07-23 22:05:50 +02:00
|
|
|
db-get-specifications
|
2020-09-10 14:25:19 +02:00
|
|
|
evaluation-status
|
2020-09-14 14:31:16 +02:00
|
|
|
last-insert-rowid
|
|
|
|
expect-one-row
|
2016-06-26 15:13:31 +02:00
|
|
|
db-add-evaluation
|
2020-09-10 14:25:19 +02:00
|
|
|
db-abort-pending-evaluations
|
|
|
|
db-set-evaluation-status
|
2020-09-06 13:03:08 +02:00
|
|
|
db-set-evaluation-time
|
2018-04-05 22:15:20 +02:00
|
|
|
db-get-pending-derivations
|
2018-01-23 17:37:08 +01:00
|
|
|
build-status
|
2016-07-25 20:31:06 +02:00
|
|
|
db-add-build
|
2020-06-10 15:58:11 +02:00
|
|
|
db-add-build-product
|
2020-09-29 14:34:14 +02:00
|
|
|
db-register-builds
|
2018-01-23 18:15:42 +01:00
|
|
|
db-update-build-status!
|
2020-01-04 10:43:04 +01:00
|
|
|
db-get-output
|
2020-04-17 14:45:13 +02:00
|
|
|
db-get-inputs
|
2017-07-31 19:25:28 +02:00
|
|
|
db-get-build
|
|
|
|
db-get-builds
|
2020-11-24 17:54:31 +01:00
|
|
|
db-get-time-since-previous-build
|
2019-05-17 22:33:27 +02:00
|
|
|
db-get-builds-by-search
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
db-get-builds-min
|
|
|
|
db-get-builds-max
|
2019-05-17 22:33:27 +02:00
|
|
|
db-get-builds-query-min
|
|
|
|
db-get-builds-query-max
|
2019-10-21 23:43:16 +02:00
|
|
|
db-add-event
|
|
|
|
db-get-events
|
|
|
|
db-delete-events-with-ids-<=-to
|
2020-12-07 15:34:29 +01:00
|
|
|
db-get-evaluation
|
2018-04-19 11:17:42 +02:00
|
|
|
db-get-evaluations
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
db-get-evaluations-build-summary
|
|
|
|
db-get-evaluations-id-min
|
|
|
|
db-get-evaluations-id-max
|
2018-08-05 21:25:37 +02:00
|
|
|
db-get-evaluation-specification
|
2020-06-10 15:58:11 +02:00
|
|
|
db-get-build-product-path
|
|
|
|
db-get-build-products
|
2019-06-19 10:21:33 +02:00
|
|
|
db-get-evaluation-summary
|
2020-04-17 14:45:13 +02:00
|
|
|
db-get-checkouts
|
2016-07-16 18:16:39 +02:00
|
|
|
read-sql-file
|
2016-07-25 20:20:53 +02:00
|
|
|
read-quoted-string
|
2020-09-14 14:31:16 +02:00
|
|
|
%sqlite-exec
|
2016-07-22 12:35:09 +02:00
|
|
|
sqlite-exec
|
2020-10-14 16:28:59 +02:00
|
|
|
catch-sqlite-error
|
|
|
|
;; Constants.
|
|
|
|
SQLITE_CONSTRAINT_PRIMARYKEY
|
|
|
|
SQLITE_CONSTRAINT_UNIQUE
|
2016-06-26 18:40:31 +02:00
|
|
|
;; Parameters.
|
|
|
|
%package-database
|
2016-07-23 17:00:38 +02:00
|
|
|
%package-schema-file
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
%db-channel
|
2020-10-14 16:28:59 +02:00
|
|
|
%db-writer-channel
|
2019-10-21 23:43:16 +02:00
|
|
|
%record-events?
|
2016-06-26 18:40:31 +02:00
|
|
|
;; Macros.
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
with-db-worker-thread
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
with-db-writer-worker-thread
|
|
|
|
with-db-writer-worker-thread/force
|
2020-09-30 11:56:04 +02:00
|
|
|
with-database
|
2020-10-14 13:49:03 +02:00
|
|
|
with-queue-writer-worker))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
2018-02-08 17:31:39 +01:00
|
|
|
(define (%sqlite-exec db sql . args)
|
database: Use 'sqlite-bind' to avoid SQL injection.
* src/cuirass/database.scm (%sqlite-exec): Remove.
(sqlite-exec): Turn back into a procedure. Use 'sqlite-bind'. Add
'normalize' procedure and use it.
(db-add-specification, db-add-derivation, db-get-derivation)
(db-add-evaluation, db-add-build, db-update-build-status!)
(db-get-build, db-get-stamp, db-add-stamp): Use question marks in SQL
queries.
* src/cuirass/base.scm (build-packages)[register]: Make #:log
non-false.
* tests/database.scm (make-dummy-job): Add #:job-name, #:system,
#:nix-name, and #:eval-id. This is necessary because 'sqlite-bind'
would now translate #f to a real NULL (before it would translate to the
string "#f"...), and would thus report violations of the non-NULL
constraint.
Co-authored-by: Ludovic Courtès <ludo@gnu.org>
2018-02-08 11:39:45 +01:00
|
|
|
"Evaluate the given SQL query with the given ARGS. Return the list of
|
|
|
|
rows."
|
|
|
|
(define (normalize arg)
|
|
|
|
;; Turn ARG into a string, unless it's a primitive SQL datatype.
|
|
|
|
(if (or (null? arg) (pair? arg) (vector? arg))
|
|
|
|
(object->string arg)
|
|
|
|
arg))
|
|
|
|
|
2018-02-08 11:46:29 +01:00
|
|
|
(let ((stmt (sqlite-prepare db sql #:cache? #t)))
|
database: Use 'sqlite-bind' to avoid SQL injection.
* src/cuirass/database.scm (%sqlite-exec): Remove.
(sqlite-exec): Turn back into a procedure. Use 'sqlite-bind'. Add
'normalize' procedure and use it.
(db-add-specification, db-add-derivation, db-get-derivation)
(db-add-evaluation, db-add-build, db-update-build-status!)
(db-get-build, db-get-stamp, db-add-stamp): Use question marks in SQL
queries.
* src/cuirass/base.scm (build-packages)[register]: Make #:log
non-false.
* tests/database.scm (make-dummy-job): Add #:job-name, #:system,
#:nix-name, and #:eval-id. This is necessary because 'sqlite-bind'
would now translate #f to a real NULL (before it would translate to the
string "#f"...), and would thus report violations of the non-NULL
constraint.
Co-authored-by: Ludovic Courtès <ludo@gnu.org>
2018-02-08 11:39:45 +01:00
|
|
|
(for-each (lambda (arg index)
|
|
|
|
(sqlite-bind stmt index (normalize arg)))
|
|
|
|
args (iota (length args) 1))
|
|
|
|
(let ((result (sqlite-fold-right cons '() stmt)))
|
2020-12-06 11:57:44 +01:00
|
|
|
(sqlite-reset stmt)
|
database: Use 'sqlite-bind' to avoid SQL injection.
* src/cuirass/database.scm (%sqlite-exec): Remove.
(sqlite-exec): Turn back into a procedure. Use 'sqlite-bind'. Add
'normalize' procedure and use it.
(db-add-specification, db-add-derivation, db-get-derivation)
(db-add-evaluation, db-add-build, db-update-build-status!)
(db-get-build, db-get-stamp, db-add-stamp): Use question marks in SQL
queries.
* src/cuirass/base.scm (build-packages)[register]: Make #:log
non-false.
* tests/database.scm (make-dummy-job): Add #:job-name, #:system,
#:nix-name, and #:eval-id. This is necessary because 'sqlite-bind'
would now translate #f to a real NULL (before it would translate to the
string "#f"...), and would thus report violations of the non-NULL
constraint.
Co-authored-by: Ludovic Courtès <ludo@gnu.org>
2018-02-08 11:39:45 +01:00
|
|
|
result)))
|
2018-01-25 10:38:19 +01:00
|
|
|
|
2018-02-08 17:31:39 +01:00
|
|
|
(define-syntax sqlite-exec/bind
|
|
|
|
(lambda (s)
|
|
|
|
;; Expand to an '%sqlite-exec' call where the query string has
|
|
|
|
;; interspersed question marks and the argument list is separate.
|
|
|
|
(define (string-literal? s)
|
|
|
|
(string? (syntax->datum s)))
|
|
|
|
|
|
|
|
(syntax-case s ()
|
|
|
|
((_ db (bindings ...) tail str arg rest ...)
|
|
|
|
#'(sqlite-exec/bind db
|
|
|
|
(bindings ... (str arg))
|
|
|
|
tail
|
|
|
|
rest ...))
|
|
|
|
((_ db (bindings ...) tail str)
|
|
|
|
#'(sqlite-exec/bind db (bindings ...) str))
|
|
|
|
((_ db ((strings args) ...) tail)
|
|
|
|
(and (every string-literal? #'(strings ...))
|
|
|
|
(string-literal? #'tail))
|
|
|
|
;; Optimized case: only string literals.
|
|
|
|
(with-syntax ((query (string-join
|
|
|
|
(append (syntax->datum #'(strings ...))
|
|
|
|
(list (syntax->datum #'tail)))
|
|
|
|
"? ")))
|
|
|
|
#'(%sqlite-exec db query args ...)))
|
|
|
|
((_ db ((strings args) ...) tail)
|
|
|
|
;; Fallback case: some of the strings aren't literals.
|
|
|
|
#'(%sqlite-exec db (string-join (list strings ... tail) "? ")
|
|
|
|
args ...)))))
|
|
|
|
|
|
|
|
(define-syntax-rule (sqlite-exec db query args ...)
|
|
|
|
"Execute the specific QUERY with the given ARGS. Uses of 'sqlite-exec'
|
|
|
|
typically look like this:
|
|
|
|
|
|
|
|
(sqlite-exec db \"SELECT * FROM Foo WHERE x = \"
|
|
|
|
x \"AND Y=\" y \";\")
|
|
|
|
|
|
|
|
References to variables 'x' and 'y' here are replaced by question marks in the
|
|
|
|
SQL query, and then 'sqlite-bind' is used to bind them.
|
|
|
|
|
|
|
|
This ensures that (1) SQL injection is impossible, and (2) the number of
|
|
|
|
question marks matches the number of arguments to bind."
|
|
|
|
(sqlite-exec/bind db () "" query args ...))
|
|
|
|
|
2018-11-16 21:47:18 +01:00
|
|
|
(define-syntax catch-sqlite-error
|
|
|
|
(syntax-rules (on =>)
|
|
|
|
"Run EXP..., catching SQLite error and handling the given code as
|
|
|
|
specified."
|
|
|
|
((_ exp ... (on error => handle ...))
|
|
|
|
(catch 'sqlite-error
|
|
|
|
(lambda ()
|
|
|
|
exp ...)
|
|
|
|
(lambda (key who code message . rest)
|
|
|
|
(if (= code error)
|
|
|
|
(begin handle ...)
|
2018-11-16 21:49:02 +01:00
|
|
|
(apply throw key who code message rest)))))))
|
2018-11-16 21:47:18 +01:00
|
|
|
|
2016-06-26 18:40:31 +02:00
|
|
|
(define %package-database
|
|
|
|
;; Define to the database file name of this package.
|
2018-08-16 18:13:26 +02:00
|
|
|
(make-parameter (string-append %localstatedir "/lib/" %package
|
2018-01-22 14:51:14 +01:00
|
|
|
"/" %package ".db")))
|
2016-06-26 18:40:31 +02:00
|
|
|
|
2016-07-16 18:16:39 +02:00
|
|
|
(define %package-schema-file
|
|
|
|
;; Define to the database schema file of this package.
|
|
|
|
(make-parameter (string-append (or (getenv "CUIRASS_DATADIR")
|
|
|
|
(string-append %datadir "/" %package))
|
|
|
|
"/schema.sql")))
|
|
|
|
|
2018-07-07 00:31:14 +02:00
|
|
|
(define %package-sql-dir
|
|
|
|
;; Define to the directory containing the SQL files.
|
|
|
|
(make-parameter (string-append (or (getenv "CUIRASS_DATADIR")
|
|
|
|
(string-append %datadir "/" %package))
|
|
|
|
"/sql")))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define %db-channel
|
|
|
|
(make-parameter #f))
|
|
|
|
|
2020-10-14 13:49:03 +02:00
|
|
|
(define %db-writer-channel
|
2020-09-30 11:56:04 +02:00
|
|
|
(make-parameter #f))
|
|
|
|
|
2019-10-21 23:43:16 +02:00
|
|
|
(define %record-events?
|
|
|
|
(make-parameter #f))
|
|
|
|
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(define-syntax-rule (with-db-worker-thread db exp ...)
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
"Evaluate EXP... in the critical section corresponding to %DB-CHANNEL.
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
DB is bound to the argument of that critical section: the database connection.
|
|
|
|
This must only be used for reading queries, i.e SELECT queries."
|
2020-10-03 10:14:33 +02:00
|
|
|
(let ((send-timeout 2)
|
|
|
|
(receive-timeout 5)
|
2020-10-03 08:26:36 +02:00
|
|
|
(caller-name (frame-procedure-name
|
|
|
|
(stack-ref (make-stack #t) 1))))
|
2020-08-01 11:56:43 +02:00
|
|
|
(call-with-worker-thread
|
|
|
|
(%db-channel)
|
|
|
|
(lambda (db) exp ...)
|
2020-10-03 10:14:33 +02:00
|
|
|
#:send-timeout send-timeout
|
|
|
|
#:send-timeout-proc
|
|
|
|
(lambda ()
|
|
|
|
(log-message
|
|
|
|
(format #f "No available database workers for ~a seconds."
|
|
|
|
(number->string send-timeout))))
|
|
|
|
#:receive-timeout receive-timeout
|
|
|
|
#:receive-timeout-proc
|
2020-08-01 11:56:43 +02:00
|
|
|
(lambda ()
|
|
|
|
(log-message
|
2020-10-03 08:26:36 +02:00
|
|
|
(format #f "Database worker unresponsive for ~a seconds (~a)."
|
2020-10-03 10:14:33 +02:00
|
|
|
(number->string receive-timeout)
|
2020-10-03 08:26:36 +02:00
|
|
|
caller-name))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-10-14 13:49:03 +02:00
|
|
|
(define-syntax with-db-writer-worker-thread
|
|
|
|
(syntax-rules ()
|
|
|
|
"Similar to WITH-DB-WORKER-THREAD but evaluates EXP in a database worker
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
dedicated to writing. EXP evaluation is deferred and will only be run once
|
|
|
|
the worker evaluation queue in full. To force an immediate evaluation the
|
|
|
|
#:FORCE? option or the alias below may be used. This macro is reserved for
|
|
|
|
writing queries, i.e CREATE, DELETE, DROP, INSERT, or UPDATE queries."
|
2020-10-14 13:49:03 +02:00
|
|
|
((_ db #:force? force exp ...)
|
|
|
|
(call-with-worker-thread
|
|
|
|
(%db-writer-channel)
|
|
|
|
(lambda (db) exp ...)
|
|
|
|
#:options `((#:force? . ,force))))
|
|
|
|
((_ db exp ...)
|
|
|
|
(with-db-writer-worker-thread db #:force? #f exp ...))))
|
2020-09-29 14:34:14 +02:00
|
|
|
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(define-syntax with-db-writer-worker-thread/force
|
|
|
|
(syntax-rules ()
|
|
|
|
"Alias for WITH-DB-WRITER-WORKER-THREAD with FORCE? option set."
|
|
|
|
((_ db exp ...)
|
|
|
|
(with-db-writer-worker-thread db #:force? #t exp ...))))
|
|
|
|
|
2016-07-16 18:16:39 +02:00
|
|
|
(define (read-sql-file file-name)
|
|
|
|
"Return a list of string containing SQL instructions from FILE-NAME."
|
|
|
|
(call-with-input-file file-name
|
2017-07-06 10:03:16 +02:00
|
|
|
(lambda (port)
|
2016-07-16 18:16:39 +02:00
|
|
|
(let loop ((insts '()))
|
|
|
|
(let ((inst (read-delimited ";" port 'concat)))
|
|
|
|
(if (or (eof-object? inst)
|
|
|
|
;; Don't cons the spaces after the last instructions.
|
|
|
|
(string-every char-whitespace? inst))
|
|
|
|
(reverse! insts)
|
|
|
|
(loop (cons inst insts))))))))
|
|
|
|
|
2018-03-25 00:02:16 +01:00
|
|
|
(define (set-db-options db)
|
|
|
|
"Set various options for DB and return it."
|
|
|
|
|
|
|
|
;; Turn DB in "write-ahead log" mode and return it.
|
2018-01-26 18:20:30 +01:00
|
|
|
(sqlite-exec db "PRAGMA journal_mode=WAL;")
|
2018-03-25 00:02:16 +01:00
|
|
|
|
|
|
|
;; Install a busy handler such that, when the database is locked, sqlite
|
|
|
|
;; retries until 30 seconds have passed, at which point it gives up and
|
|
|
|
;; throws SQLITE_BUSY. This is useful when we have several fibers or
|
|
|
|
;; threads accessing the database concurrently.
|
|
|
|
;;(sqlite-busy-timeout db (* 30 1000))
|
|
|
|
(sqlite-exec db "PRAGMA busy_timeout = 30000;")
|
|
|
|
|
2020-10-08 19:42:23 +02:00
|
|
|
;; The want to prioritize read operations over write operations as we can
|
|
|
|
;; have a large number of clients, while the number of write operations is
|
|
|
|
;; modest. Use a small WAL journal to do that, and try to reduce disk I/O
|
|
|
|
;; by increasing RAM usage as described here:
|
|
|
|
;; https://wiki.mozilla.org/Performance/Avoid_SQLite_In_Your_Next_Firefox_Feature
|
|
|
|
(sqlite-exec db "PRAGMA wal_autocheckpoint = 16;")
|
|
|
|
(sqlite-exec db "PRAGMA journal_size_limit = 1536;")
|
|
|
|
(sqlite-exec db "PRAGMA page_size = 32768;")
|
|
|
|
(sqlite-exec db "PRAGMA cache_size = -500000;")
|
|
|
|
(sqlite-exec db "PRAGMA temp_store = MEMORY;")
|
|
|
|
(sqlite-exec db "PRAGMA synchronous = NORMAL;")
|
2018-01-26 18:20:30 +01:00
|
|
|
db)
|
|
|
|
|
2018-07-07 00:31:14 +02:00
|
|
|
(define (db-load db schema)
|
|
|
|
"Evaluate the file SCHEMA, which may contain SQL queries, into DB."
|
|
|
|
(for-each (cut sqlite-exec db <>)
|
|
|
|
(read-sql-file schema)))
|
|
|
|
|
|
|
|
(define (db-schema-version db)
|
|
|
|
(vector-ref (car (sqlite-exec db "PRAGMA user_version;")) 0))
|
|
|
|
|
|
|
|
(define (db-set-schema-version db version)
|
|
|
|
(sqlite-exec db (format #f "PRAGMA user_version = ~d;" version)))
|
|
|
|
|
|
|
|
(define (latest-db-schema-version)
|
|
|
|
"Return the version to which the schema should be upgraded, based on the
|
|
|
|
upgrade-n.sql files, or 0 if there are no such files."
|
|
|
|
(reduce max 0
|
|
|
|
(map (compose string->number (cut match:substring <> 1))
|
|
|
|
(filter-map (cut string-match "^upgrade-([0-9]+)\\.sql$" <>)
|
|
|
|
(or (scandir (%package-sql-dir)) '())))))
|
|
|
|
|
2016-07-23 22:04:03 +02:00
|
|
|
(define* (db-init #:optional (db-name (%package-database))
|
|
|
|
#:key (schema (%package-schema-file)))
|
2016-06-26 18:40:31 +02:00
|
|
|
"Open the database to store and read jobs and builds informations. Return a
|
|
|
|
database object."
|
2016-07-23 22:04:03 +02:00
|
|
|
(when (file-exists? db-name)
|
|
|
|
(format (current-error-port) "Removing leftover database ~a~%" db-name)
|
|
|
|
(delete-file db-name))
|
|
|
|
(let ((db (sqlite-open db-name (logior SQLITE_OPEN_CREATE
|
2018-07-19 12:48:44 +02:00
|
|
|
SQLITE_OPEN_READWRITE
|
|
|
|
SQLITE_OPEN_NOMUTEX))))
|
2018-07-07 00:31:14 +02:00
|
|
|
(db-load db schema)
|
|
|
|
(db-set-schema-version db (latest-db-schema-version))
|
2016-07-23 22:04:03 +02:00
|
|
|
db))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
2018-07-07 00:31:14 +02:00
|
|
|
(define (schema-upgrade-file version)
|
|
|
|
"Return the file containing the SQL instructions that upgrade the schema
|
|
|
|
from VERSION-1 to VERSION."
|
|
|
|
(in-vicinity (%package-sql-dir) (format #f "upgrade-~a.sql" version)))
|
|
|
|
|
|
|
|
(define (db-upgrade db)
|
|
|
|
"Upgrade database DB based on its current version and the available
|
|
|
|
upgrade-n.sql files."
|
|
|
|
(for-each (lambda (version)
|
|
|
|
(db-load db (schema-upgrade-file version))
|
|
|
|
(db-set-schema-version db version))
|
|
|
|
(let ((current (db-schema-version db)))
|
|
|
|
(iota (- (latest-db-schema-version) current) (1+ current))))
|
|
|
|
db)
|
|
|
|
|
2016-07-26 11:06:48 +02:00
|
|
|
(define* (db-open #:optional (db (%package-database)))
|
2016-06-26 18:40:31 +02:00
|
|
|
"Open database to store or read jobs and builds informations. Return a
|
|
|
|
database object."
|
2018-01-26 18:20:30 +01:00
|
|
|
;; Use "write-ahead log" mode because it improves concurrency and should
|
|
|
|
;; avoid SQLITE_LOCKED errors when we have several readers:
|
|
|
|
;; <https://www.sqlite.org/wal.html>.
|
2018-07-19 12:48:44 +02:00
|
|
|
|
|
|
|
;; SQLITE_OPEN_NOMUTEX disables mutexing on database connection and prepared
|
|
|
|
;; statement objects, thus making us responsible for serializing access to
|
|
|
|
;; database connections and prepared statements.
|
2018-03-25 00:02:16 +01:00
|
|
|
(set-db-options (if (file-exists? db)
|
2018-07-19 12:48:44 +02:00
|
|
|
(db-upgrade
|
|
|
|
(sqlite-open db (logior SQLITE_OPEN_READWRITE
|
|
|
|
SQLITE_OPEN_NOMUTEX)))
|
2018-03-25 00:02:16 +01:00
|
|
|
(db-init db))))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
|
|
|
(define (db-close db)
|
|
|
|
"Close database object DB."
|
|
|
|
(sqlite-close db))
|
|
|
|
|
2020-05-25 12:08:20 +02:00
|
|
|
(define* (db-optimize #:optional (db-file (%package-database)))
|
|
|
|
"Open the database and perform optimizations."
|
|
|
|
(let ((db (db-open db-file)))
|
|
|
|
(sqlite-exec db "PRAGMA optimize;")
|
|
|
|
(sqlite-exec db "PRAGMA wal_checkpoint(TRUNCATE);")
|
|
|
|
(db-close db)))
|
|
|
|
|
2020-09-22 15:17:31 +02:00
|
|
|
(define (trace-callback trace p x)
|
|
|
|
(log-query (pointer->string
|
|
|
|
(sqlite-expanded-sql p))
|
|
|
|
(make-time 'time-duration
|
|
|
|
(bytevector-uint-ref
|
|
|
|
(pointer->bytevector x (sizeof uint64))
|
|
|
|
0 (native-endianness)
|
|
|
|
(sizeof uint64))
|
|
|
|
0)))
|
|
|
|
|
|
|
|
(define (db-log-queries file)
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(query-logging-port (open-output-file file))
|
|
|
|
(sqlite-trace db SQLITE_TRACE_PROFILE trace-callback)))
|
|
|
|
|
2016-07-22 13:13:52 +02:00
|
|
|
(define (last-insert-rowid db)
|
|
|
|
(vector-ref (car (sqlite-exec db "SELECT last_insert_rowid();"))
|
|
|
|
0))
|
|
|
|
|
2019-10-21 23:43:16 +02:00
|
|
|
(define (changes-count db)
|
|
|
|
"The number of database rows that were changed or inserted or deleted by the
|
|
|
|
most recently completed INSERT, DELETE, or UPDATE statement."
|
|
|
|
(vector-ref (car (sqlite-exec db "SELECT changes();"))
|
|
|
|
0))
|
|
|
|
|
2018-09-30 17:26:12 +02:00
|
|
|
(define (expect-one-row rows)
|
|
|
|
"Several SQL queries expect one result, or zero if not found. This gets rid
|
|
|
|
of the list, and returns #f when there is no result."
|
|
|
|
(match rows
|
|
|
|
((row) row)
|
|
|
|
(() #f)))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-add-input spec-name input)
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(sqlite-exec db "\
|
Add support for multiple inputs.
* Makefile.am (dist_sql_DATA): Add src/sql/upgrade-1.sql.
* bin/cuirass.in (show-help, %options, main): Remove the LOAD-PATH option that
was used afterwards as %GUIX-PACKAGE-PATH.
* bin/evaluate.in (absolutize, input-checkout, spec-source, spec-load-path,
spec-package-path, format-checkouts): New procedures.
(%not-colon): Remove variable.
(main): Take the load path, package path and PROC from the checkouts that
result from the inputs. Format the checkouts before sending them to the
procedure. Remove the LOAD-PATH argument.
* doc/cuirass.texi (Overview, Database schema): Document the changes.
* examples/{guix-jobs.scm, hello-git.scm, hello-singleton.scm,
hello-subset.scm, random.scm}: Adapt to the new specification format.
* examples/guix-track-git.scm (package->spec): Rename to PACKAGE->INPUT.
(package->git-tracked): Replace FETCH-REPOSITORY with FETCH-INPUT and handle
the new format of its return value.
* examples/random-jobs.scm (make-random-jobs): Rename RANDOM to CHECKOUT.
Rename the checkout from 'random (which is a specification) to 'cuirass (which
is a checkout resulting from an input).
* src/cuirass/base.scm (fetch-repository): Rename to fetch-input. Rename SPEC
to INPUT. Return a checkout object instead of returning two values.
(evaluate): Take a list of CHECKOUTS and COMMITS as arguments, instead of
SOURCE. Remove TOKENIZE and LOAD-PATH. Pass the CHECKOUTS instead of the
SOURCE to "evaluate". Remove %GUIX-PACKAGE-PATH. Build the EVAL object
instead of getting it from "evaluate".
(compile?, fetch-inputs, compile-checkouts): New procedures.
(process-specs): Fetch all inputs instead of only fetching one repository.
The result of that fetching operation is a list of CHECKOUTS whose COMMITS are
used as a STAMP.
(%guix-package-path, set-guix-package-path): Remove them.
* src/cuirass/database.scm (db-add-input, db-get-inputs): New procedures.
(db-add-specification, db-get-specifications): Adapt to the new specification
format. Add/get all inputs as well.
(db-add-evaluation): Rename REVISION to COMMITS. Store COMMITS as space
separated commit hashes.
(db-get-builds): Rename REPO_NAME to NAME.
(db-get-stamp): Rename COMMIT to STAMP. Return #f when there is no STAMP.
(db-add-stamp): Rename COMMIT to STAMP. Deal with DB-GET-STAMP's new return
value.
(db-get-evaluations): Rename REVISION to COMMITS. Tokenize COMMITS.
* src/cuirass/utils.scm (%non-blocking): Export it.
* src/schema.sql (Inputs): New table that refers to the Specifications table.
(Specifications): Move input related fields to the Inputs table. Rename
REPO_NAME to NAME. Rename ARGUMENTS to PROC_ARGS. Rename FILE to PROC_FILE.
Add LOAD_PATH_INPUTS, PACKAGE_PATH_INPUTS and PROC_INPUT fields that refer to
the Inputs table.
(Stamps): Rename REPO_NAME to NAME.
(Evaluations): Rename REPO_NAME to NAME. Rename REVISION to COMMITS.
(Specifications_index): Replace with Inputs_index.
* src/sql/upgrade-1.sql: New file.
* tests/database.scm (example-spec, make-dummy-eval, sqlite-exec): Adapt to
the new specifications format. Rename REVISION to COMMITS.
* tests/http.scm (evaluations-query-result, fill-db): Idem.
2018-06-26 11:18:23 +02:00
|
|
|
INSERT OR IGNORE INTO Inputs (specification, name, url, load_path, branch, \
|
|
|
|
tag, revision, no_compile_p) VALUES ("
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
spec-name ", "
|
|
|
|
(assq-ref input #:name) ", "
|
|
|
|
(assq-ref input #:url) ", "
|
|
|
|
(assq-ref input #:load-path) ", "
|
|
|
|
(assq-ref input #:branch) ", "
|
|
|
|
(assq-ref input #:tag) ", "
|
|
|
|
(assq-ref input #:commit) ", "
|
2020-01-24 20:46:25 +01:00
|
|
|
(if (assq-ref input #:no-compile?) 1 0) ");")))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(define (db-add-checkout spec-name eval-id checkout)
|
|
|
|
"Insert CHECKOUT associated with SPEC-NAME and EVAL-ID. If a checkout with
|
|
|
|
the same revision already exists for SPEC-NAME, return #f."
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2018-11-16 21:47:18 +01:00
|
|
|
(catch-sqlite-error
|
|
|
|
(sqlite-exec db "\
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
INSERT INTO Checkouts (specification, revision, evaluation, input,
|
2020-09-06 13:03:08 +02:00
|
|
|
directory, timestamp) VALUES ("
|
2018-11-16 21:47:18 +01:00
|
|
|
spec-name ", "
|
|
|
|
(assq-ref checkout #:commit) ", "
|
|
|
|
eval-id ", "
|
|
|
|
(assq-ref checkout #:input) ", "
|
2020-09-06 13:03:08 +02:00
|
|
|
(assq-ref checkout #:directory) ", "
|
|
|
|
(or (assq-ref checkout #:timestamp) 0) ");")
|
2018-11-16 21:47:18 +01:00
|
|
|
(last-insert-rowid db)
|
|
|
|
|
|
|
|
;; If we get a unique-constraint-failed error, that means we have
|
|
|
|
;; already inserted the same checkout. That happens for each input
|
|
|
|
;; that doesn't change between two evaluations.
|
|
|
|
(on SQLITE_CONSTRAINT_PRIMARYKEY => #f))))
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-add-specification spec)
|
|
|
|
"Store SPEC in database the database. SPEC inputs are stored in the INPUTS
|
|
|
|
table."
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(sqlite-exec db "\
|
Add support for multiple inputs.
* Makefile.am (dist_sql_DATA): Add src/sql/upgrade-1.sql.
* bin/cuirass.in (show-help, %options, main): Remove the LOAD-PATH option that
was used afterwards as %GUIX-PACKAGE-PATH.
* bin/evaluate.in (absolutize, input-checkout, spec-source, spec-load-path,
spec-package-path, format-checkouts): New procedures.
(%not-colon): Remove variable.
(main): Take the load path, package path and PROC from the checkouts that
result from the inputs. Format the checkouts before sending them to the
procedure. Remove the LOAD-PATH argument.
* doc/cuirass.texi (Overview, Database schema): Document the changes.
* examples/{guix-jobs.scm, hello-git.scm, hello-singleton.scm,
hello-subset.scm, random.scm}: Adapt to the new specification format.
* examples/guix-track-git.scm (package->spec): Rename to PACKAGE->INPUT.
(package->git-tracked): Replace FETCH-REPOSITORY with FETCH-INPUT and handle
the new format of its return value.
* examples/random-jobs.scm (make-random-jobs): Rename RANDOM to CHECKOUT.
Rename the checkout from 'random (which is a specification) to 'cuirass (which
is a checkout resulting from an input).
* src/cuirass/base.scm (fetch-repository): Rename to fetch-input. Rename SPEC
to INPUT. Return a checkout object instead of returning two values.
(evaluate): Take a list of CHECKOUTS and COMMITS as arguments, instead of
SOURCE. Remove TOKENIZE and LOAD-PATH. Pass the CHECKOUTS instead of the
SOURCE to "evaluate". Remove %GUIX-PACKAGE-PATH. Build the EVAL object
instead of getting it from "evaluate".
(compile?, fetch-inputs, compile-checkouts): New procedures.
(process-specs): Fetch all inputs instead of only fetching one repository.
The result of that fetching operation is a list of CHECKOUTS whose COMMITS are
used as a STAMP.
(%guix-package-path, set-guix-package-path): Remove them.
* src/cuirass/database.scm (db-add-input, db-get-inputs): New procedures.
(db-add-specification, db-get-specifications): Adapt to the new specification
format. Add/get all inputs as well.
(db-add-evaluation): Rename REVISION to COMMITS. Store COMMITS as space
separated commit hashes.
(db-get-builds): Rename REPO_NAME to NAME.
(db-get-stamp): Rename COMMIT to STAMP. Return #f when there is no STAMP.
(db-add-stamp): Rename COMMIT to STAMP. Deal with DB-GET-STAMP's new return
value.
(db-get-evaluations): Rename REVISION to COMMITS. Tokenize COMMITS.
* src/cuirass/utils.scm (%non-blocking): Export it.
* src/schema.sql (Inputs): New table that refers to the Specifications table.
(Specifications): Move input related fields to the Inputs table. Rename
REPO_NAME to NAME. Rename ARGUMENTS to PROC_ARGS. Rename FILE to PROC_FILE.
Add LOAD_PATH_INPUTS, PACKAGE_PATH_INPUTS and PROC_INPUT fields that refer to
the Inputs table.
(Stamps): Rename REPO_NAME to NAME.
(Evaluations): Rename REPO_NAME to NAME. Rename REVISION to COMMITS.
(Specifications_index): Replace with Inputs_index.
* src/sql/upgrade-1.sql: New file.
* tests/database.scm (example-spec, make-dummy-eval, sqlite-exec): Adapt to
the new specifications format. Rename REVISION to COMMITS.
* tests/http.scm (evaluations-query-result, fill-db): Idem.
2018-06-26 11:18:23 +02:00
|
|
|
INSERT OR IGNORE INTO Specifications (name, load_path_inputs, \
|
2020-06-10 15:58:11 +02:00
|
|
|
package_path_inputs, proc_input, proc_file, proc, proc_args, \
|
|
|
|
build_outputs) \
|
2018-02-08 17:31:39 +01:00
|
|
|
VALUES ("
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(assq-ref spec #:name) ", "
|
|
|
|
(assq-ref spec #:load-path-inputs) ", "
|
|
|
|
(assq-ref spec #:package-path-inputs) ", "
|
|
|
|
(assq-ref spec #:proc-input) ", "
|
|
|
|
(assq-ref spec #:proc-file) ", "
|
|
|
|
(symbol->string (assq-ref spec #:proc)) ", "
|
2020-06-10 15:58:11 +02:00
|
|
|
(assq-ref spec #:proc-args) ", "
|
|
|
|
(assq-ref spec #:build-outputs) ");")
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let ((spec-id (last-insert-rowid db)))
|
|
|
|
(for-each (lambda (input)
|
|
|
|
(db-add-input (assq-ref spec #:name) input))
|
|
|
|
(assq-ref spec #:inputs))
|
|
|
|
spec-id)))
|
|
|
|
|
2019-10-30 09:19:28 +01:00
|
|
|
(define (db-remove-specification name)
|
|
|
|
"Remove the specification matching NAME from the database and its inputs."
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2019-10-30 09:19:28 +01:00
|
|
|
(sqlite-exec db "BEGIN TRANSACTION;")
|
|
|
|
(sqlite-exec db "\
|
|
|
|
DELETE FROM Inputs WHERE specification=" name ";")
|
|
|
|
(sqlite-exec db "\
|
|
|
|
DELETE FROM Specifications WHERE name=" name ";")
|
|
|
|
(sqlite-exec db "COMMIT;")))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-inputs spec-name)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let loop ((rows (sqlite-exec
|
|
|
|
db "SELECT * FROM Inputs WHERE specification="
|
|
|
|
spec-name ";"))
|
|
|
|
(inputs '()))
|
|
|
|
(match rows
|
|
|
|
(() inputs)
|
|
|
|
((#(specification name url load-path branch tag revision no-compile-p)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:name . ,name)
|
|
|
|
(#:url . ,url)
|
|
|
|
(#:load-path . ,load-path)
|
|
|
|
(#:branch . ,branch)
|
|
|
|
(#:tag . ,tag)
|
|
|
|
(#:commit . ,revision)
|
|
|
|
(#:no-compile? . ,(positive? no-compile-p)))
|
|
|
|
inputs)))))))
|
|
|
|
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
(define (db-get-specification name)
|
|
|
|
"Retrieve a specification in the database with the given NAME."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
Fix spec reading when restarting builds.
When "spawn-builds" is called to restart builds, the spec is not known,
preventing build products from being created as reported here:
https://issues.guix.gnu.org/42523
Fix this issue by reading the specification in database in
"set-build-successful!" procedure.
* src/cuirass/database.scm (db-get-specification): New exported procedure,
(db-get-specifications): add an optional name argument.
* tests/database.scm (db-get-specification): Add a corresponding test-case.
* src/cuirass/base.scm (set-build-successful!): Remove spec argument and read
it directly from database instead,
(update-build-statuses!): also remove spec argument, adapt
set-build-successful! call accordingly,
(spawn-builds): remove spec argument and adapt handle-build-event and
update-build-statuses! calls accordingly,
(handle-build-event): remove spec argument, adapt
set-build-successful! call accordingly,
(build-packages): remove spec argument, adapt spawn-builds call accordingly,
(process-specs): adapt build-packages call.
2020-07-25 14:22:20 +02:00
|
|
|
(expect-one-row (db-get-specifications name))))
|
|
|
|
|
|
|
|
(define* (db-get-specifications #:optional name)
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(let loop
|
|
|
|
((rows (if name
|
|
|
|
(sqlite-exec db "
|
|
|
|
SELECT * FROM Specifications WHERE name =" name ";")
|
|
|
|
(sqlite-exec db "
|
|
|
|
SELECT * FROM Specifications ORDER BY name DESC;")))
|
|
|
|
(specs '()))
|
|
|
|
(match rows
|
|
|
|
(() specs)
|
|
|
|
((#(name load-path-inputs package-path-inputs proc-input proc-file proc
|
|
|
|
proc-args build-outputs)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:name . ,name)
|
|
|
|
(#:load-path-inputs .
|
|
|
|
,(with-input-from-string load-path-inputs read))
|
|
|
|
(#:package-path-inputs .
|
|
|
|
,(with-input-from-string package-path-inputs read))
|
|
|
|
(#:proc-input . ,proc-input)
|
|
|
|
(#:proc-file . ,proc-file)
|
|
|
|
(#:proc . ,(with-input-from-string proc read))
|
|
|
|
(#:proc-args . ,(with-input-from-string proc-args read))
|
|
|
|
(#:inputs . ,(db-get-inputs name))
|
|
|
|
(#:build-outputs .
|
|
|
|
,(with-input-from-string build-outputs read)))
|
|
|
|
specs)))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-09-10 14:25:19 +02:00
|
|
|
(define-enumeration evaluation-status
|
|
|
|
(started -1)
|
|
|
|
(succeeded 0)
|
|
|
|
(failed 1)
|
|
|
|
(aborted 2))
|
|
|
|
|
2020-09-06 13:03:08 +02:00
|
|
|
(define* (db-add-evaluation spec-name checkouts
|
|
|
|
#:key
|
|
|
|
(checkouttime 0)
|
|
|
|
(evaltime 0)
|
|
|
|
timestamp)
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
"Add a new evaluation for SPEC-NAME only if one of the CHECKOUTS is new.
|
|
|
|
Otherwise, return #f."
|
2020-09-06 13:03:08 +02:00
|
|
|
(define now
|
|
|
|
(or timestamp (time-second (current-time time-utc))))
|
|
|
|
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(sqlite-exec db "BEGIN TRANSACTION;")
|
2020-09-10 14:25:19 +02:00
|
|
|
(sqlite-exec db "INSERT INTO Evaluations (specification, status,
|
2020-09-06 13:03:08 +02:00
|
|
|
timestamp, checkouttime, evaltime)
|
2020-09-10 14:25:19 +02:00
|
|
|
VALUES (" spec-name "," (evaluation-status started) ","
|
|
|
|
now "," checkouttime "," evaltime ");")
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(let* ((eval-id (last-insert-rowid db))
|
|
|
|
(new-checkouts (filter-map
|
|
|
|
(cut db-add-checkout spec-name eval-id <>)
|
|
|
|
checkouts)))
|
|
|
|
(if (null? new-checkouts)
|
|
|
|
(begin (sqlite-exec db "ROLLBACK;")
|
|
|
|
#f)
|
2019-10-28 08:56:51 +01:00
|
|
|
(begin (db-add-event 'evaluation
|
|
|
|
(time-second (current-time time-utc))
|
|
|
|
`((#:evaluation . ,eval-id)
|
|
|
|
(#:specification . ,spec-name)
|
|
|
|
(#:in_progress . #t)))
|
|
|
|
(sqlite-exec db "COMMIT;")
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
eval-id)))))
|
|
|
|
|
2020-09-10 14:25:19 +02:00
|
|
|
(define (db-abort-pending-evaluations)
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2020-09-10 14:25:19 +02:00
|
|
|
(sqlite-exec db "UPDATE Evaluations SET status =
|
|
|
|
" (evaluation-status aborted) " WHERE status = "
|
|
|
|
(evaluation-status started))))
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
|
2020-09-10 14:25:19 +02:00
|
|
|
(define (db-set-evaluation-status eval-id status)
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2020-09-10 14:25:19 +02:00
|
|
|
(sqlite-exec db "UPDATE Evaluations SET status =
|
|
|
|
" status " WHERE id = " eval-id ";")))
|
2016-06-26 15:13:31 +02:00
|
|
|
|
2020-09-06 13:03:08 +02:00
|
|
|
(define (db-set-evaluation-time eval-id)
|
|
|
|
(define now
|
|
|
|
(time-second (current-time time-utc)))
|
|
|
|
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force
|
2020-09-06 13:03:08 +02:00
|
|
|
db
|
|
|
|
(sqlite-exec db "UPDATE Evaluations SET evaltime = " now
|
|
|
|
"WHERE id = " eval-id ";")))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define-syntax-rule (with-database body ...)
|
2020-01-24 20:04:21 +01:00
|
|
|
"Run BODY with %DB-CHANNEL being dynamically bound to a channel providing a
|
2020-09-30 11:56:04 +02:00
|
|
|
worker thread that allows database operations to run without interfering with
|
2020-01-24 20:04:21 +01:00
|
|
|
fibers."
|
2020-09-30 11:56:04 +02:00
|
|
|
(parameterize ((%db-channel
|
|
|
|
(make-worker-thread-channel
|
|
|
|
(lambda ()
|
|
|
|
(list (db-open)))
|
|
|
|
#:parallelism
|
|
|
|
(min (current-processor-count) 4))))
|
|
|
|
body ...))
|
|
|
|
|
2020-10-14 13:49:03 +02:00
|
|
|
(define-syntax-rule (with-queue-writer-worker body ...)
|
|
|
|
"Run BODY with %DB-WRITER-CHANNEL being dynamically bound to a channel
|
|
|
|
providing a worker thread that allow database write operations to run
|
|
|
|
without interfering with fibers.
|
|
|
|
|
|
|
|
The worker will queue write operations and run them in a single transaction
|
|
|
|
when the queue is full. As write operations are exclusive in SQLite, do not
|
|
|
|
allocate more than one worker."
|
|
|
|
(parameterize ((%db-writer-channel
|
2020-09-30 11:56:04 +02:00
|
|
|
(make-worker-thread-channel
|
|
|
|
(lambda ()
|
|
|
|
(list (db-open)))
|
2020-10-14 13:49:03 +02:00
|
|
|
#:parallelism 1
|
|
|
|
#:queue-size 100
|
|
|
|
#:queue-proc
|
|
|
|
(lambda (db run-queue)
|
|
|
|
(log-message "Running writer queue.")
|
|
|
|
(sqlite-exec db "BEGIN TRANSACTION;")
|
|
|
|
(run-queue)
|
|
|
|
(sqlite-exec db "COMMIT;")))))
|
2020-01-24 20:04:21 +01:00
|
|
|
body ...))
|
2016-06-26 18:40:31 +02:00
|
|
|
|
2016-07-25 20:20:53 +02:00
|
|
|
(define* (read-quoted-string #:optional (port (current-input-port)))
|
2016-06-29 16:16:48 +02:00
|
|
|
"Read all of the characters out of PORT and return them as a SQL quoted
|
|
|
|
string."
|
|
|
|
(let loop ((chars '()))
|
|
|
|
(let ((char (read-char port)))
|
|
|
|
(cond ((eof-object? char) (list->string (reverse! chars)))
|
|
|
|
((char=? char #\') (loop (cons* char char chars)))
|
|
|
|
(else (loop (cons char chars)))))))
|
|
|
|
|
2017-08-26 10:42:40 +02:00
|
|
|
;; Extended error codes (see <sqlite3.h>).
|
|
|
|
;; XXX: This should be defined by (sqlite3).
|
|
|
|
(define SQLITE_CONSTRAINT 19)
|
|
|
|
(define SQLITE_CONSTRAINT_PRIMARYKEY
|
|
|
|
(logior SQLITE_CONSTRAINT (ash 6 8)))
|
2018-02-14 16:40:50 +01:00
|
|
|
(define SQLITE_CONSTRAINT_UNIQUE
|
|
|
|
(logior SQLITE_CONSTRAINT (ash 8 8)))
|
2017-08-26 10:42:40 +02:00
|
|
|
|
2018-01-23 17:37:08 +01:00
|
|
|
(define-enumeration build-status
|
2018-01-23 18:15:42 +01:00
|
|
|
;; Build status as expected by Hydra's API. Note: the negative values are
|
|
|
|
;; Cuirass' own extensions.
|
|
|
|
(scheduled -2)
|
|
|
|
(started -1)
|
2018-01-23 17:37:08 +01:00
|
|
|
(succeeded 0)
|
|
|
|
(failed 1)
|
|
|
|
(failed-dependency 2)
|
|
|
|
(failed-other 3)
|
2018-01-23 23:16:32 +01:00
|
|
|
(canceled 4))
|
2018-01-23 17:37:08 +01:00
|
|
|
|
2018-09-02 09:45:48 +02:00
|
|
|
(define (db-add-output derivation output)
|
|
|
|
"Insert OUTPUT associated with DERIVATION. If an output with the same path
|
|
|
|
already exists, return #f."
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2018-11-16 21:47:18 +01:00
|
|
|
(catch-sqlite-error
|
|
|
|
(match output
|
|
|
|
((name . path)
|
|
|
|
(sqlite-exec db "\
|
2018-09-02 09:45:48 +02:00
|
|
|
INSERT INTO Outputs (derivation, name, path) VALUES ("
|
2018-11-16 21:47:18 +01:00
|
|
|
derivation ", " name ", " path ");")))
|
|
|
|
(last-insert-rowid db)
|
|
|
|
|
|
|
|
;; If we get a unique-constraint-failed error, that means we have
|
|
|
|
;; already inserted the same output. That happens with fixed-output
|
|
|
|
;; derivations.
|
|
|
|
(on SQLITE_CONSTRAINT_PRIMARYKEY => #f))))
|
2018-09-02 09:45:48 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-add-build build)
|
2018-09-02 09:45:48 +02:00
|
|
|
"Store BUILD in database the database only if one of its outputs is new.
|
|
|
|
Return #f otherwise. BUILD outputs are stored in the OUTPUTS table."
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2020-10-08 12:02:40 +02:00
|
|
|
(sqlite-exec db "
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
INSERT INTO Builds (derivation, evaluation, job_name, system, nix_name, log,
|
|
|
|
status, timestamp, starttime, stoptime)
|
|
|
|
VALUES ("
|
2020-10-08 12:02:40 +02:00
|
|
|
(assq-ref build #:derivation) ", "
|
|
|
|
(assq-ref build #:eval-id) ", "
|
|
|
|
(assq-ref build #:job-name) ", "
|
|
|
|
(assq-ref build #:system) ", "
|
|
|
|
(assq-ref build #:nix-name) ", "
|
|
|
|
(assq-ref build #:log) ", "
|
|
|
|
(or (assq-ref build #:status)
|
|
|
|
(build-status scheduled)) ", "
|
|
|
|
(or (assq-ref build #:timestamp) 0) ", "
|
|
|
|
(or (assq-ref build #:starttime) 0) ", "
|
|
|
|
(or (assq-ref build #:stoptime) 0) ");")
|
|
|
|
(let* ((derivation (assq-ref build #:derivation))
|
|
|
|
(outputs (assq-ref build #:outputs))
|
|
|
|
(new-outputs (filter-map (cut db-add-output derivation <>)
|
|
|
|
outputs)))
|
|
|
|
(db-add-event 'build
|
|
|
|
(assq-ref build #:timestamp)
|
|
|
|
`((#:derivation . ,(assq-ref build #:derivation))
|
|
|
|
;; TODO Ideally this would use the value
|
|
|
|
;; from build, with a default of scheduled,
|
|
|
|
;; but it's hard to convert to the symbol,
|
|
|
|
;; so just hard code scheduled for now.
|
|
|
|
(#:event . scheduled)))
|
|
|
|
derivation)))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-06-10 15:58:11 +02:00
|
|
|
(define (db-add-build-product product)
|
|
|
|
"Insert PRODUCT into BuildProducts table."
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2020-06-10 15:58:11 +02:00
|
|
|
(sqlite-exec db "\
|
2020-06-30 11:46:58 +02:00
|
|
|
INSERT OR IGNORE INTO BuildProducts (build, type, file_size, checksum,
|
2020-06-10 15:58:11 +02:00
|
|
|
path) VALUES ("
|
|
|
|
(assq-ref product #:build) ", "
|
|
|
|
(assq-ref product #:type) ", "
|
|
|
|
(assq-ref product #:file-size) ", "
|
|
|
|
(assq-ref product #:checksum) ", "
|
|
|
|
(assq-ref product #:path) ");")
|
|
|
|
(last-insert-rowid db)))
|
|
|
|
|
2020-10-01 18:18:33 +02:00
|
|
|
(define (db-register-builds jobs eval-id)
|
2020-09-29 14:34:14 +02:00
|
|
|
(define (new-outputs? outputs)
|
|
|
|
(let ((new-outputs
|
|
|
|
(filter-map (match-lambda
|
|
|
|
((name . path)
|
|
|
|
(let ((drv (db-get-output path)))
|
|
|
|
(and (not drv) path))))
|
|
|
|
outputs)))
|
|
|
|
(not (null? new-outputs))))
|
|
|
|
|
|
|
|
(define (register job)
|
|
|
|
(let* ((name (assq-ref job #:job-name))
|
|
|
|
(drv (assq-ref job #:derivation))
|
|
|
|
(job-name (assq-ref job #:job-name))
|
|
|
|
(system (assq-ref job #:system))
|
|
|
|
(nix-name (assq-ref job #:nix-name))
|
2020-10-01 18:18:33 +02:00
|
|
|
(log (assq-ref job #:log))
|
2020-11-24 17:54:31 +01:00
|
|
|
(period (assq-ref job #:period))
|
2020-10-01 18:18:33 +02:00
|
|
|
(outputs (assq-ref job #:outputs))
|
2020-09-29 14:34:14 +02:00
|
|
|
(cur-time (time-second (current-time time-utc))))
|
|
|
|
(and (new-outputs? outputs)
|
|
|
|
(let ((build `((#:derivation . ,drv)
|
|
|
|
(#:eval-id . ,eval-id)
|
|
|
|
(#:job-name . ,job-name)
|
|
|
|
(#:system . ,system)
|
|
|
|
(#:nix-name . ,nix-name)
|
|
|
|
|
|
|
|
;; XXX: We'd leave LOG to #f (i.e., NULL) but that
|
|
|
|
;; currently violates the non-NULL constraint.
|
|
|
|
(#:log . ,(or log ""))
|
|
|
|
|
|
|
|
(#:status . ,(build-status scheduled))
|
|
|
|
(#:outputs . ,outputs)
|
|
|
|
(#:timestamp . ,cur-time)
|
|
|
|
(#:starttime . 0)
|
|
|
|
(#:stoptime . 0))))
|
2020-11-24 17:54:31 +01:00
|
|
|
(if period
|
2020-11-25 15:49:04 +01:00
|
|
|
(let* ((spec (db-get-evaluation-specification eval-id))
|
|
|
|
(time
|
|
|
|
(db-get-time-since-previous-build job-name spec))
|
2020-11-24 17:54:31 +01:00
|
|
|
(add-build? (cond
|
|
|
|
((not time) #t)
|
|
|
|
((> time period) #t)
|
|
|
|
(else #f))))
|
|
|
|
(and add-build? (db-add-build build)))
|
|
|
|
(db-add-build build))))))
|
2020-09-29 14:34:14 +02:00
|
|
|
|
2020-10-14 13:49:03 +02:00
|
|
|
;; Use the database worker dedicated to write queries. We don't want this
|
|
|
|
;; query to be queued as it is already a quite large transaction by itself,
|
|
|
|
;; so pass the #:FORCE? option.
|
2020-10-15 10:45:13 +02:00
|
|
|
(with-db-writer-worker-thread/force db
|
2020-10-01 18:18:33 +02:00
|
|
|
(log-message "Registering builds for evaluation ~a." eval-id)
|
2020-10-08 12:02:40 +02:00
|
|
|
(sqlite-exec db "BEGIN TRANSACTION;")
|
|
|
|
(let ((derivations (filter-map register jobs)))
|
|
|
|
(sqlite-exec db "COMMIT;")
|
|
|
|
derivations)))
|
2020-09-29 14:34:14 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define* (db-update-build-status! drv status #:key log-file)
|
|
|
|
"Update the database so that DRV's status is STATUS. This also updates the
|
2018-01-24 23:40:07 +01:00
|
|
|
'starttime' or 'stoptime' fields. If LOG-FILE is true, record it as the build
|
|
|
|
log file for DRV."
|
2018-01-23 18:15:42 +01:00
|
|
|
(define now
|
|
|
|
(time-second (current-time time-utc)))
|
|
|
|
|
2019-10-21 23:43:16 +02:00
|
|
|
(define status-names
|
|
|
|
`((,(build-status succeeded) . "succeeded")
|
|
|
|
(,(build-status failed) . "failed")
|
|
|
|
(,(build-status failed-dependency) . "failed (dependency)")
|
|
|
|
(,(build-status failed-other) . "failed (other)")
|
|
|
|
(,(build-status canceled) . "canceled")))
|
|
|
|
|
2020-10-14 13:49:03 +02:00
|
|
|
(with-db-writer-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(if (= status (build-status started))
|
2019-10-21 23:43:16 +02:00
|
|
|
(begin
|
|
|
|
(sqlite-exec db "UPDATE Builds SET starttime=" now ", status="
|
|
|
|
status "WHERE derivation=" drv ";")
|
|
|
|
(db-add-event 'build
|
|
|
|
now
|
|
|
|
`((#:derivation . ,drv)
|
|
|
|
(#:event . started))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
|
|
|
;; Update only if we're switching to a different status; otherwise
|
|
|
|
;; leave things unchanged. This ensures that 'stoptime' remains valid
|
|
|
|
;; and doesn't change every time we mark DRV as 'succeeded' several
|
|
|
|
;; times in a row, for instance.
|
2019-10-21 23:43:16 +02:00
|
|
|
(begin
|
|
|
|
(if log-file
|
|
|
|
(sqlite-exec db "UPDATE Builds SET stoptime=" now
|
|
|
|
", status=" status ", log=" log-file
|
|
|
|
"WHERE derivation=" drv "AND status != " status ";")
|
|
|
|
(sqlite-exec db "UPDATE Builds SET stoptime=" now
|
|
|
|
", status=" status
|
|
|
|
"WHERE derivation=" drv " AND status != " status
|
|
|
|
";"))
|
|
|
|
(when (positive? (changes-count db))
|
|
|
|
(db-add-event 'build
|
|
|
|
now
|
|
|
|
`((#:derivation . ,drv)
|
|
|
|
(#:event . ,(assq-ref status-names
|
|
|
|
status)))))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-01-04 10:43:04 +01:00
|
|
|
(define (db-get-output path)
|
|
|
|
"Retrieve the OUTPUT for PATH."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2020-01-04 10:43:04 +01:00
|
|
|
;; There isn't a unique index on path, but because Cuirass avoids adding
|
|
|
|
;; derivations which introduce the same outputs, there should only be one
|
|
|
|
;; result.
|
|
|
|
(match (sqlite-exec db "SELECT derivation, name FROM Outputs
|
|
|
|
WHERE path =" path "
|
|
|
|
LIMIT 1;")
|
|
|
|
(() #f)
|
|
|
|
((#(derivation name))
|
|
|
|
`((#:derivation . ,derivation)
|
|
|
|
(#:name . ,name))))))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-outputs derivation)
|
|
|
|
"Retrieve the OUTPUTS of the build identified by DERIVATION in the
|
|
|
|
database."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let loop ((rows
|
|
|
|
(sqlite-exec db "SELECT name, path FROM Outputs
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
WHERE derivation =" derivation ";"))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(outputs '()))
|
|
|
|
(match rows
|
|
|
|
(() outputs)
|
|
|
|
((#(name path)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `(,name . ((#:path . ,path)))
|
|
|
|
outputs)))))))
|
2017-07-31 19:25:28 +02:00
|
|
|
|
2019-05-21 11:40:28 +02:00
|
|
|
(define (query->bind-arguments query-string)
|
|
|
|
"Return a list of keys to query strings by parsing QUERY-STRING."
|
2020-04-15 19:50:25 +02:00
|
|
|
(define status-values
|
|
|
|
`(("success" . ,(build-status succeeded))
|
|
|
|
("failed" . ,(build-status failed))
|
|
|
|
("failed-dependency" . ,(build-status failed-dependency))
|
|
|
|
("failed-other" . ,(build-status failed-other))
|
|
|
|
("canceled" . ,(build-status canceled))))
|
2019-06-20 23:31:34 +02:00
|
|
|
(let ((args (append-map
|
|
|
|
(lambda (token)
|
|
|
|
(match (string-split token #\:)
|
|
|
|
(("system" system)
|
|
|
|
`(#:system ,system))
|
|
|
|
(("spec" spec)
|
|
|
|
`(#:spec ,spec))
|
2020-04-15 19:50:25 +02:00
|
|
|
(("status" status)
|
|
|
|
`(#:status ,(assoc-ref status-values status)))
|
2019-06-20 23:31:34 +02:00
|
|
|
((_ invalid) '()) ; ignore
|
|
|
|
((query)
|
2020-10-05 14:46:31 +02:00
|
|
|
;; Remove any '%' that could make the search too slow and
|
|
|
|
;; add one at the end of the query.
|
|
|
|
`(#:query ,(string-append
|
|
|
|
(string-join
|
|
|
|
(string-split query #\%)
|
|
|
|
"")
|
|
|
|
"%")))))
|
2019-06-20 23:31:34 +02:00
|
|
|
(string-tokenize query-string))))
|
2019-05-21 11:40:28 +02:00
|
|
|
;; Normalize arguments
|
|
|
|
(fold (lambda (key acc)
|
|
|
|
(if (member key acc)
|
|
|
|
acc
|
|
|
|
(append (list key #f) acc)))
|
|
|
|
args '(#:spec #:system))))
|
|
|
|
|
2019-05-17 22:33:27 +02:00
|
|
|
(define (db-get-builds-by-search filters)
|
|
|
|
"Retrieve all builds in the database which are matched by given FILTERS.
|
|
|
|
FILTERS is an assoc list whose possible keys are the symbols query,
|
|
|
|
border-low-id, border-high-id, and nr."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2020-10-05 14:46:31 +02:00
|
|
|
(let* ((stmt-text (format #f "SELECT Builds.rowid, Builds.timestamp,
|
|
|
|
Builds.starttime,Builds.stoptime, Builds.log, Builds.status,
|
|
|
|
Builds.job_name, Builds.system, Builds.nix_name, Specifications.name
|
2019-05-17 22:33:27 +02:00
|
|
|
FROM Builds
|
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
|
|
|
WHERE (Builds.nix_name LIKE :query)
|
2020-04-15 19:50:25 +02:00
|
|
|
AND (:status IS NULL
|
|
|
|
OR (Builds.status = :status))
|
2019-05-21 11:40:28 +02:00
|
|
|
AND (:spec IS NULL
|
|
|
|
OR (Specifications.name = :spec))
|
|
|
|
AND (:system IS NULL
|
|
|
|
OR (Builds.system = :system))
|
2019-05-17 22:33:27 +02:00
|
|
|
AND (:borderlowid IS NULL
|
|
|
|
OR (:borderlowid < Builds.rowid))
|
|
|
|
AND (:borderhighid IS NULL
|
|
|
|
OR (:borderhighid > Builds.rowid))
|
|
|
|
ORDER BY
|
|
|
|
CASE WHEN :borderlowid IS NULL THEN Builds.rowid
|
|
|
|
ELSE -Builds.rowid
|
|
|
|
END DESC
|
2020-10-05 14:46:31 +02:00
|
|
|
LIMIT :nr;"))
|
2019-05-17 22:33:27 +02:00
|
|
|
(stmt (sqlite-prepare db stmt-text #:cache? #t)))
|
2019-05-21 11:40:28 +02:00
|
|
|
(apply sqlite-bind-arguments
|
|
|
|
stmt
|
|
|
|
(append (list
|
|
|
|
#:borderlowid (assq-ref filters 'border-low-id)
|
|
|
|
#:borderhighid (assq-ref filters 'border-high-id)
|
|
|
|
#:nr (match (assq-ref filters 'nr)
|
|
|
|
(#f -1)
|
|
|
|
(x x)))
|
|
|
|
(query->bind-arguments (assq-ref filters 'query))))
|
2020-12-06 11:58:21 +01:00
|
|
|
(let ((builds
|
|
|
|
(sqlite-fold-right
|
|
|
|
(lambda (row result)
|
|
|
|
(match row
|
|
|
|
(#(id timestamp starttime stoptime log status job-name
|
|
|
|
system nix-name specification)
|
|
|
|
(cons `((#:id . ,id)
|
|
|
|
(#:timestamp . ,timestamp)
|
|
|
|
(#:starttime . ,starttime)
|
|
|
|
(#:stoptime . ,stoptime)
|
|
|
|
(#:log . ,log)
|
|
|
|
(#:status . ,status)
|
|
|
|
(#:job-name . ,job-name)
|
|
|
|
(#:system . ,system)
|
|
|
|
(#:nix-name . ,nix-name)
|
|
|
|
(#:specification . ,specification)
|
|
|
|
(#:buildproducts . ,(db-get-build-products id)))
|
|
|
|
result))))
|
|
|
|
'()
|
|
|
|
stmt)))
|
|
|
|
(sqlite-reset stmt)
|
|
|
|
builds))))
|
2019-05-17 22:33:27 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-builds filters)
|
|
|
|
"Retrieve all builds in the database which are matched by given FILTERS.
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
FILTERS is an assoc list whose possible keys are 'derivation | 'id | 'jobset |
|
|
|
|
'job | 'system | 'nr | 'order | 'status | 'evaluation."
|
2020-05-25 16:12:07 +02:00
|
|
|
|
2020-10-07 11:49:30 +02:00
|
|
|
;; XXX: Make sure that all filters are covered by an index.
|
2020-05-25 16:12:07 +02:00
|
|
|
(define (filters->order filters)
|
2020-12-02 17:16:08 +01:00
|
|
|
(lambda (inner)
|
|
|
|
(match (assq 'order filters)
|
|
|
|
(('order . 'build-id) "Builds.rowid ASC")
|
|
|
|
(('order . 'finish-time) "stoptime DESC")
|
|
|
|
(('order . 'finish-time+build-id)
|
|
|
|
(if inner
|
|
|
|
"CASE WHEN :borderlowid IS NULL THEN
|
|
|
|
stoptime ELSE -stoptime END DESC,
|
|
|
|
CASE WHEN :borderlowid IS NULL THEN
|
|
|
|
Builds.rowid ELSE -Builds.rowid END DESC"
|
|
|
|
"stoptime DESC, Builds.rowid DESC"))
|
|
|
|
;; With this order, builds in 'running' state (-1) appear
|
|
|
|
;; before those in 'scheduled' state (-2).
|
|
|
|
(('order . 'status+submission-time)
|
|
|
|
"Builds.status DESC, Builds.timestamp DESC, Builds.rowid ASC")
|
|
|
|
(_ "Builds.rowid DESC"))))
|
2020-05-25 16:12:07 +02:00
|
|
|
|
2020-10-07 11:49:30 +02:00
|
|
|
;; XXX: Make sure that all filters are covered by an index.
|
2020-05-25 16:12:07 +02:00
|
|
|
(define (where-conditions filters)
|
|
|
|
(define filter-name->sql
|
|
|
|
`((id . "Builds.id = :id")
|
|
|
|
(jobset . "Specifications.name = :jobset")
|
|
|
|
(derivation . "Builds.derivation = :derivation")
|
|
|
|
(job . "Builds.job_name = :job")
|
|
|
|
(system . "Builds.system = :system")
|
|
|
|
(evaluation . "Builds.evaluation = :evaluation")
|
|
|
|
(status . ,(match (assq-ref filters 'status)
|
|
|
|
(#f #f)
|
|
|
|
('done "Builds.status >= 0")
|
2020-07-02 21:07:21 +02:00
|
|
|
('started "Builds.status = -1")
|
2020-05-25 16:12:07 +02:00
|
|
|
('pending "Builds.status < 0")
|
|
|
|
('succeeded "Builds.status = 0")
|
|
|
|
('failed "Builds.status > 0")))
|
2020-12-02 17:16:08 +01:00
|
|
|
(border-low-time
|
|
|
|
. "(:borderlowtime IS NULL OR :borderlowid IS NULL OR
|
|
|
|
((:borderlowtime, :borderlowid) < (Builds.stoptime, Builds.rowid)))")
|
|
|
|
(border-high-time
|
|
|
|
. "(:borderhightime IS NULL OR :borderhighid IS NULL OR
|
|
|
|
((:borderhightime, :borderhighid) > (Builds.stoptime, Builds.rowid)))")))
|
2020-05-25 16:12:07 +02:00
|
|
|
|
|
|
|
(filter
|
|
|
|
string?
|
|
|
|
(fold
|
|
|
|
(lambda (filter-name where-condition-parts)
|
|
|
|
(if (assq-ref filters filter-name)
|
|
|
|
(cons (assq-ref filter-name->sql filter-name)
|
|
|
|
where-condition-parts)
|
|
|
|
where-condition-parts))
|
|
|
|
'()
|
|
|
|
(map car filters))))
|
|
|
|
|
2020-10-07 11:49:30 +02:00
|
|
|
(define (format-outputs names paths)
|
|
|
|
(map (lambda (name path)
|
|
|
|
`(,name . ((#:path . ,path))))
|
|
|
|
(string-split names #\,)
|
|
|
|
(string-split paths #\,)))
|
|
|
|
|
|
|
|
(define (format-build-products ids types file-sizes checksums paths)
|
|
|
|
(define (split list)
|
|
|
|
(if list
|
|
|
|
(string-split list #\,)
|
|
|
|
'()))
|
|
|
|
|
|
|
|
(map (lambda (id type file-size checksum path)
|
|
|
|
`((#:id . ,(string->number id))
|
|
|
|
(#:type . ,type)
|
|
|
|
(#:file-size . ,(string->number file-size))
|
|
|
|
(#:checksum . ,checksum)
|
|
|
|
(#:path . ,path)))
|
|
|
|
(split ids)
|
|
|
|
(split types)
|
|
|
|
(split file-sizes)
|
|
|
|
(split checksums)
|
|
|
|
(split paths)))
|
|
|
|
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let* ((order (filters->order filters))
|
2020-05-25 16:12:07 +02:00
|
|
|
(where (match (where-conditions filters)
|
|
|
|
(() "")
|
|
|
|
((condition)
|
|
|
|
(string-append "WHERE " condition "\n"))
|
|
|
|
((first-condition rest ...)
|
|
|
|
(string-append "WHERE " first-condition "\n AND "
|
|
|
|
(string-join rest " AND ")))))
|
|
|
|
(stmt-text
|
|
|
|
(format #f "
|
2020-10-07 11:49:30 +02:00
|
|
|
SELECT Builds.*,
|
|
|
|
GROUP_CONCAT(Outputs.name), GROUP_CONCAT(Outputs.path),
|
|
|
|
GROUP_CONCAT(BP.rowid), GROUP_CONCAT(BP.type), GROUP_CONCAT(BP.file_size),
|
|
|
|
GROUP_CONCAT(BP.checksum), GROUP_CONCAT(BP.path) FROM
|
|
|
|
(SELECT Builds.derivation, Builds.rowid, Builds.timestamp, Builds.starttime,
|
|
|
|
Builds.stoptime, Builds.log, Builds.status, Builds.job_name,
|
|
|
|
Builds.system, Builds.nix_name, Builds.evaluation,
|
|
|
|
Specifications.name
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
FROM Builds
|
database: Merge Derivations into Builds table.
Fixes <https://bugs.gnu.org/32190>.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-2.sql'.
* doc/cuirass.texi (Derivations): Remove section.
(Builds): Update accordingly. Add columns from the Derivations table.
(Outputs): Replace 'id' with 'derivation'.
* src/cuirass/base.scm (evaluate): Don't add jobs to the Derivations table.
(build-packages): Add columns that were in the Derivations table before. Only
build the derivations that were successfully registered, that is, those that
didn't exist in the Builds table. Give a derivation instead of a build id to
DB-GET-BUILD. Compute the number of failed jobs based on the derivations that
were added to the table, instead of the jobs.
* src/cuirass/database.scm (db-add-derivation, db-get-derivation): Remove
exported procedures.
(db-add-build): Catch SQLITE_CONSTRAINT_PRIMARYKEY error, which means that two
jobs produced the same derivation, and return #f in that case. Add columns
that were in the Derivations table before. Use 'derivation' as primary key
for the Outputs table.
(db-get-outputs): Use 'derivation' as identifier, instead of 'build-id'.
(filters->order): Replace 'id' with 'rowid'.
(db-get-builds): Add a 'derivation' filter. Replace 'id' with 'rowid'.
Remove the 'INNER JOIN Derivations'. Replace Derivations with Builds. Return
'derivation' in first position to make it clear that it's the primary key.
Pass DERIVATION instead of ID to DB-GET-OUTPUTS.
(db-get-build): Allow to take a derivation as argument. Use NUMBER? to
differentiate between derivations and ids.
(db-get-pending-derivations): Remove the 'SELECT DISTINCT' clause now that
derivations are unique. Remove the 'INNER JOIN Builds'.
(db-get-evaluations-build-summary, db-get-builds-min, db-get-builds-max):
Replace 'id' with 'rowid'.
* src/schema.sql (Derivations): Remove table.
(Outputs): Replace Builds.id with Builds.derivation.
(Builds): Use 'derivation' as primary key. Remove the 'id' column. Add
'job_name', 'system', 'nix_name' columns that were in the Derivations table
before.
(Builds_Derivations_index): Rename to Builds_index. Update accordingly.
(Derivations_index): Remove index.
* src/sql/upgrade-2.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-job, make-dummy-derivation): Remove
procedures.
(make-dummy-build): Add columns that were in MAKE-DUMMY-DERIVATION. Get the
DRV parameter to be mandatory because it's a primary key.
(%id): Remove parameter.
("db-add-derivation", "db-get-derivation"): Remove tests.
("db-add-build"): Expect #f, because it adds twice the same derivation. Pass
the derivation argument to MAKE-DUMMY-BUILD.
("db-update-build-status!"): Rename 'id' to 'derivation'. Pass the derivation
argument to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION call.
("db-get-builds", "db-get-pending-derivations"): Pass the derivation argument
to MAKE-DUMMY-BUILD. Remove the DB-ADD-DERIVATION calls.
* tests/http.scm ("fill-db"): Remove DERIVATION1 and DERIVATION2, and put
their content in BUILD1 and BUILD2. Remove the DB-ADD-DERIVATION calls.
2018-08-01 00:03:12 +02:00
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
2020-05-25 16:12:07 +02:00
|
|
|
~a
|
|
|
|
ORDER BY ~a
|
2020-10-07 11:49:30 +02:00
|
|
|
LIMIT :nr) Builds
|
|
|
|
INNER JOIN Outputs ON Outputs.derivation = Builds.derivation
|
|
|
|
LEFT JOIN BuildProducts as BP ON BP.build = Builds.rowid
|
|
|
|
GROUP BY Builds.derivation
|
|
|
|
ORDER BY ~a;"
|
2020-12-02 17:16:08 +01:00
|
|
|
where (order #t) (order #f)))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(stmt (sqlite-prepare db stmt-text #:cache? #t)))
|
2020-05-25 16:12:07 +02:00
|
|
|
|
|
|
|
(sqlite-bind stmt 'nr (match (assq-ref filters 'nr)
|
|
|
|
(#f -1)
|
|
|
|
(x x)))
|
|
|
|
(for-each (match-lambda
|
|
|
|
(('nr . _) #f) ; Handled above
|
|
|
|
(('order . _) #f) ; Doesn't need binding
|
|
|
|
(('status . _) #f) ; Doesn't need binding
|
|
|
|
((name . value)
|
|
|
|
(when value
|
|
|
|
(sqlite-bind stmt
|
|
|
|
(or (assq-ref
|
|
|
|
'((border-low-time . borderlowtime)
|
|
|
|
(border-high-time . borderhightime)
|
|
|
|
(border-low-id . borderlowid)
|
|
|
|
(border-high-id . borderhighid))
|
|
|
|
name)
|
|
|
|
name)
|
|
|
|
value))))
|
|
|
|
filters)
|
2020-12-06 11:58:21 +01:00
|
|
|
(let ((builds
|
|
|
|
(sqlite-fold-right
|
|
|
|
(lambda (row result)
|
|
|
|
(match row
|
|
|
|
(#(derivation id timestamp starttime stoptime log status job-name
|
|
|
|
system nix-name eval-id specification
|
|
|
|
outputs-name outputs-path
|
|
|
|
products-id products-type products-file-size
|
|
|
|
products-checksum products-path)
|
|
|
|
(cons `((#:derivation . ,derivation)
|
|
|
|
(#:id . ,id)
|
|
|
|
(#:timestamp . ,timestamp)
|
|
|
|
(#:starttime . ,starttime)
|
|
|
|
(#:stoptime . ,stoptime)
|
|
|
|
(#:log . ,log)
|
|
|
|
(#:status . ,status)
|
|
|
|
(#:job-name . ,job-name)
|
|
|
|
(#:system . ,system)
|
|
|
|
(#:nix-name . ,nix-name)
|
|
|
|
(#:eval-id . ,eval-id)
|
|
|
|
(#:specification . ,specification)
|
|
|
|
(#:outputs . ,(format-outputs outputs-name
|
|
|
|
outputs-path))
|
|
|
|
(#:buildproducts .
|
|
|
|
,(format-build-products products-id
|
|
|
|
products-type
|
|
|
|
products-file-size
|
|
|
|
products-checksum
|
|
|
|
products-path)))
|
|
|
|
result))))
|
|
|
|
'()
|
|
|
|
stmt)))
|
|
|
|
(sqlite-reset stmt)
|
|
|
|
builds))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
|
|
|
(define (db-get-build derivation-or-id)
|
|
|
|
"Retrieve a build in the database which corresponds to DERIVATION-OR-ID."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let ((key (if (number? derivation-or-id) 'id 'derivation)))
|
2018-09-30 17:26:12 +02:00
|
|
|
(expect-one-row (db-get-builds `((,key . ,derivation-or-id)))))))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
|
2020-11-25 15:49:04 +01:00
|
|
|
(define (db-get-time-since-previous-build job-name specification)
|
2020-11-24 17:54:31 +01:00
|
|
|
"Return the time difference in seconds between the current time and the
|
2020-11-25 15:49:04 +01:00
|
|
|
registration time of the last build for JOB-NAME and SPECIFICATION."
|
2020-11-24 17:54:31 +01:00
|
|
|
(with-db-worker-thread db
|
|
|
|
(let ((rows (sqlite-exec db "
|
2020-11-25 15:49:04 +01:00
|
|
|
SELECT strftime('%s', 'now') - Builds.timestamp FROM Builds
|
|
|
|
INNER JOIN Evaluations on Builds.evaluation = Evaluations.id
|
|
|
|
WHERE job_name = " job-name "AND specification = " specification
|
|
|
|
"ORDER BY Builds.timestamp DESC LIMIT 1")))
|
2020-11-24 17:54:31 +01:00
|
|
|
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
|
|
|
|
2019-10-21 23:43:16 +02:00
|
|
|
(define (db-add-event type timestamp details)
|
|
|
|
(when (%record-events?)
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread db
|
2019-10-21 23:43:16 +02:00
|
|
|
(sqlite-exec db "\
|
|
|
|
INSERT INTO Events (type, timestamp, event_json) VALUES ("
|
|
|
|
(symbol->string type) ", "
|
|
|
|
timestamp ", "
|
|
|
|
(object->json-string details)
|
|
|
|
");")
|
|
|
|
#t)))
|
|
|
|
|
|
|
|
(define (db-get-events filters)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2019-10-21 23:43:16 +02:00
|
|
|
(let* ((stmt-text "\
|
|
|
|
SELECT Events.id,
|
|
|
|
Events.type,
|
|
|
|
Events.timestamp,
|
|
|
|
Events.event_json
|
|
|
|
FROM Events
|
|
|
|
WHERE (:type IS NULL OR (:type = Events.type))
|
|
|
|
AND (:borderlowtime IS NULL OR
|
|
|
|
:borderlowid IS NULL OR
|
|
|
|
((:borderlowtime, :borderlowid) <
|
|
|
|
(Events.timestamp, Events.id)))
|
|
|
|
AND (:borderhightime IS NULL OR
|
|
|
|
:borderhighid IS NULL OR
|
|
|
|
((:borderhightime, :borderhighid) >
|
|
|
|
(Events.timestamp, Events.id)))
|
|
|
|
ORDER BY Events.id ASC
|
|
|
|
LIMIT :nr;")
|
|
|
|
(stmt (sqlite-prepare db stmt-text #:cache? #t)))
|
|
|
|
(sqlite-bind-arguments
|
|
|
|
stmt
|
|
|
|
#:type (and=> (assq-ref filters 'type)
|
|
|
|
symbol->string)
|
|
|
|
#:nr (match (assq-ref filters 'nr)
|
|
|
|
(#f -1)
|
|
|
|
(x x)))
|
2020-12-06 11:58:21 +01:00
|
|
|
(let ((events
|
|
|
|
(sqlite-fold-right
|
|
|
|
(lambda (row result)
|
|
|
|
(match row
|
|
|
|
(#(id type timestamp event_json)
|
|
|
|
(cons `((#:id . ,id)
|
|
|
|
(#:type . ,type)
|
|
|
|
(#:timestamp . ,timestamp)
|
|
|
|
(#:event_json . ,event_json))
|
|
|
|
result))))
|
|
|
|
'()
|
|
|
|
stmt)))
|
|
|
|
(sqlite-reset stmt)
|
|
|
|
events))))
|
2019-10-21 23:43:16 +02:00
|
|
|
|
|
|
|
(define (db-delete-events-with-ids-<=-to id)
|
Use the writer worker for all write queries.
* .dir-locals.el: Add "with-queue-writer-worker".
* bin/cuirass.in: Modify "with-queue-writer-worker" scope to include the
web-server operations.
* src/cuirass/database.scm (with-db-writer-worker-thread): Export it.
(with-db-writer-worker-thread/force): New macro.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-add-evaluation, db-abort-pending-evaluations, db-set-evaluation-status,
db-set-evaluation-time, db-add-output, db-add-build-product, db-add-event,
db-delete-events-with-ids-<=-to): Use "with-db-writer-worker-thread" or
"with-db-writer-worker-thread/force" instead of "with-db-worker-thread".
* src/cuirass/metrics.scm (db-update-metrics): Ditto.
* tests/database.scm ("db-init"): Set "%db-writer-channel".
* tests/http.scm ("db-init"): Ditto.
* tests/metrics.scm ("db-init"): Ditto.
2020-10-15 09:53:53 +02:00
|
|
|
(with-db-writer-worker-thread db
|
2019-10-21 23:43:16 +02:00
|
|
|
(sqlite-exec
|
|
|
|
db
|
|
|
|
"DELETE FROM Events WHERE id <= " id ";")))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-pending-derivations)
|
2018-04-05 22:15:20 +02:00
|
|
|
"Return the list of derivation file names corresponding to pending builds in
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
the database. The returned list is guaranteed to not have any duplicates."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(map (match-lambda (#(drv) drv))
|
|
|
|
(sqlite-exec db "
|
|
|
|
SELECT derivation FROM Builds WHERE Builds.status < 0;"))))
|
|
|
|
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(define (db-get-checkouts eval-id)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(let loop ((rows (sqlite-exec
|
|
|
|
db "SELECT revision, input, directory FROM Checkouts
|
|
|
|
WHERE evaluation =" eval-id ";"))
|
|
|
|
(checkouts '()))
|
|
|
|
(match rows
|
|
|
|
(() checkouts)
|
|
|
|
((#(revision input directory)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:commit . ,revision)
|
|
|
|
(#:input . ,input)
|
|
|
|
(#:directory . ,directory))
|
|
|
|
checkouts)))))))
|
2018-04-19 11:17:42 +02:00
|
|
|
|
2020-12-07 15:34:29 +01:00
|
|
|
(define (db-get-evaluation id)
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(match (sqlite-exec db "SELECT id, specification, status,
|
|
|
|
timestamp, checkouttime, evaltime
|
|
|
|
FROM Evaluations WHERE id = " id)
|
|
|
|
(() #f)
|
|
|
|
((#(id specification status timestamp checkouttime evaltime))
|
|
|
|
`((#:id . ,id)
|
|
|
|
(#:specification . ,specification)
|
|
|
|
(#:status . ,status)
|
|
|
|
(#:timestamp . ,timestamp)
|
|
|
|
(#:checkouttime . ,checkouttime)
|
|
|
|
(#:evaltime . ,evaltime)
|
|
|
|
(#:checkouts . ,(db-get-checkouts id)))))))
|
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-evaluations limit)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2020-09-10 14:25:19 +02:00
|
|
|
(let loop ((rows (sqlite-exec db "SELECT id, specification, status,
|
2020-09-06 14:23:01 +02:00
|
|
|
timestamp, checkouttime, evaltime
|
2018-04-19 11:17:42 +02:00
|
|
|
FROM Evaluations ORDER BY id DESC LIMIT " limit ";"))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(evaluations '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse evaluations))
|
2020-09-10 14:25:19 +02:00
|
|
|
((#(id specification status timestamp checkouttime evaltime)
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,id)
|
|
|
|
(#:specification . ,specification)
|
2020-09-10 14:25:19 +02:00
|
|
|
(#:status . ,status)
|
2020-09-06 14:23:01 +02:00
|
|
|
(#:timestamp . ,timestamp)
|
|
|
|
(#:checkouttime . ,checkouttime)
|
|
|
|
(#:evaltime . ,evaltime)
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(#:checkouts . ,(db-get-checkouts id)))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
evaluations)))))))
|
|
|
|
|
|
|
|
(define (db-get-evaluations-build-summary spec limit border-low border-high)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let loop ((rows (sqlite-exec db "
|
2020-09-28 17:47:19 +02:00
|
|
|
SELECT E.id, E.status, SUM(B.status=0) as succeeded,
|
|
|
|
SUM(B.status>0) as failed, SUM(B.status<0) as scheduled FROM
|
|
|
|
(SELECT id, status FROM Evaluations
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
WHERE (specification=" spec ")
|
|
|
|
AND (" border-low "IS NULL OR (id >" border-low "))
|
|
|
|
AND (" border-high "IS NULL OR (id <" border-high "))
|
|
|
|
ORDER BY CASE WHEN " border-low "IS NULL THEN id ELSE -id END DESC
|
|
|
|
LIMIT " limit ") E
|
2020-09-28 17:47:19 +02:00
|
|
|
LEFT JOIN Builds as B
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
ON B.evaluation=E.id
|
2020-09-28 17:47:19 +02:00
|
|
|
GROUP BY E.id
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
ORDER BY E.id ASC;"))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(evaluations '()))
|
|
|
|
(match rows
|
|
|
|
(() evaluations)
|
2020-09-10 14:25:19 +02:00
|
|
|
((#(id status succeeded failed scheduled) . rest)
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,id)
|
2020-09-10 14:25:19 +02:00
|
|
|
(#:status . ,status)
|
database: Add a Checkouts table.
It is used to know when a new evaluation must be triggered and to display
input changes.
* Makefile.am (dist_sql_DATA): Add 'src/sql/upgrade-3.sql'.
* bin/cuirass.in (main): Call DB-SET-EVALUATION-DONE at startup to clear
'in-progress' evaluations.
* bin/evaluate.in (input-checkout, format-checkouts): Rename '#:name' to
'#:input'.
* doc/cuirass.texi (Stamps): Remove section.
(Checkouts): New section.
* src/cuirass/base.scm (fetch-input, fetch-inputs, compile-checkouts): Rename
'#:name' to '#:input'.
(evaluate): Remove the COMMITS argument. Add an EVAL-ID argument. Don't call
DB-ADD-EVALUATION because it was called sooner. Remove the EVAL-ID argument
to AUGMENT-JOB because it's a closure.
(build-packages): Add an EVAL-ID argument. Call DB-SET-EVALUATION-DONE once
all the derivations are registered.
(process-specs): Replace the stamping mechanism by the primary key constraint
of the Checkouts table: call "evaluate" only when DB-ADD-EVALUATION is true,
which means that at least one checkout was added. Change the EVALUATE and
BUILD-PACKAGES arguments accordingly.
* src/cuirass/database.scm (db-add-stamp, db-get-stamp): Remove procedures.
(db-set-evaluations-done, db-set-evaluation-done): New exported procedure.
(db-add-checkout): New procedure that returns #f if a checkout with the same
revision already exists.
(db-add-evaluation): Replace the EVAL argument with a SPEC-NAME and a
CHECKOUTS arguments. Insert the evaluation only if at least one checkout was
inserted. Return #f otherwise.
(db-get-checkouts): New procedure.
(db-get-evaluations, db-get-evaluations-build-summary): Handle the
'in_progress' column, remove the 'commits' column. Return the result of
DB-GET-CHECKOUTS as part of the evaluation.
* src/cuirass/templates.scm (input-changes, evaluation-badges): New
procedures.
(evaluation-info-table): Rename "Commits" to "Input changes". Use
INPUT-CHANGES to display the input changes that triggered the evaluation. Use
EVALUATION-BADGES to display a message indicating that the evaluation is in
progress.
* src/schema.sql (Stamps): Remove table.
(Checkouts): New table.
(Evaluations): Remove the 'commits' column. Add an 'in_progress' column.
* src/sql/upgrade-3.sql: New file with SQL queries to upgrade the database.
* tests/database.scm (make-dummy-eval): Remove procedure.
(make-dummy-checkouts): New procedure.
("sqlite-exec"): Remove the 'commits' column. Add the 'in_progress' column.
("db-update-build-status!", "db-get-builds", "db-get-pending-derivations"):
Update the arguments of DB-ADD-EVALUATION accordingly.
* tests/http.scm (hash-table=?): Add support for lists of hash tables.
(evaluations-query-result): Replace '#:commits' with '#:checkouts'. Return a
list instead of returning one element, for symmetry.
("fill-db"): Add a new input so that the second checkout can refer to it.
Replace EVALUATION1 and EVALUATION2 with CHECKOUTS1 and CHECKOUTS2. Update
the arguments of DB-ADD-EVALUATION accordingly.
("/api/queue?nr=100"): Take the CAR of the EVALUATIONS-QUERY-RESULT list to
make it symmetrical with the other argument of HASH-TABLE=?.
2018-08-11 20:30:11 +02:00
|
|
|
(#:checkouts . ,(db-get-checkouts id))
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(#:succeeded . ,(or succeeded 0))
|
|
|
|
(#:failed . ,(or failed 0))
|
|
|
|
(#:scheduled . ,(or scheduled 0)))
|
|
|
|
evaluations)))))))
|
|
|
|
|
|
|
|
(define (db-get-evaluations-id-min spec)
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
"Return the min id of evaluations for the given specification SPEC."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let ((rows (sqlite-exec db "
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
SELECT MIN(id) FROM Evaluations
|
|
|
|
WHERE specification=" spec)))
|
2018-09-30 17:26:12 +02:00
|
|
|
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(define (db-get-evaluations-id-max spec)
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
"Return the max id of evaluations for the given specification SPEC."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let ((rows (sqlite-exec db "
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
SELECT MAX(id) FROM Evaluations
|
|
|
|
WHERE specification=" spec)))
|
2018-09-30 17:26:12 +02:00
|
|
|
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
|
2019-06-19 10:21:33 +02:00
|
|
|
(define (db-get-evaluation-summary id)
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2019-06-19 10:21:33 +02:00
|
|
|
(let ((rows (sqlite-exec db "
|
2020-09-10 14:25:19 +02:00
|
|
|
SELECT E.id, E.status, E.timestamp, E.checkouttime, E.evaltime,
|
2020-09-28 17:47:19 +02:00
|
|
|
SUM(B.status>-100) as total, SUM(B.status=0) as succeeded,
|
|
|
|
SUM(B.status>0) as failed, SUM(B.status<0) as scheduled FROM
|
|
|
|
(SELECT id, status, timestamp, checkouttime, evaltime FROM
|
|
|
|
Evaluations WHERE (id=" id ")) E
|
|
|
|
LEFT JOIN Builds as B
|
2019-06-19 10:21:33 +02:00
|
|
|
ON B.evaluation=E.id
|
|
|
|
ORDER BY E.id ASC;")))
|
|
|
|
(and=> (expect-one-row rows)
|
|
|
|
(match-lambda
|
2020-09-10 14:25:19 +02:00
|
|
|
(#(id status timestamp checkouttime evaltime
|
2020-09-06 14:23:01 +02:00
|
|
|
total succeeded failed scheduled)
|
2019-06-19 10:21:33 +02:00
|
|
|
`((#:id . ,id)
|
2020-09-10 14:25:19 +02:00
|
|
|
(#:status . ,status)
|
2019-06-19 10:21:33 +02:00
|
|
|
(#:total . ,(or total 0))
|
2020-09-06 14:23:01 +02:00
|
|
|
(#:timestamp . ,timestamp)
|
|
|
|
(#:checkouttime . ,checkouttime)
|
|
|
|
(#:evaltime . ,evaltime)
|
2019-06-19 10:21:33 +02:00
|
|
|
(#:succeeded . ,(or succeeded 0))
|
|
|
|
(#:failed . ,(or failed 0))
|
|
|
|
(#:scheduled . ,(or scheduled 0)))))))))
|
|
|
|
|
2019-05-17 22:33:27 +02:00
|
|
|
(define (db-get-builds-query-min query)
|
|
|
|
"Return the smallest build row identifier matching QUERY."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2019-05-21 11:40:28 +02:00
|
|
|
(let* ((stmt-text "SELECT MIN(Builds.rowid) FROM Builds
|
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
|
|
|
WHERE (Builds.nix_name LIKE :query)
|
2020-04-15 19:50:25 +02:00
|
|
|
AND (:status IS NULL
|
|
|
|
OR (Builds.status = :status))
|
2019-05-21 11:40:28 +02:00
|
|
|
AND (:spec IS NULL
|
|
|
|
OR (Specifications.name = :spec))
|
|
|
|
AND (:system IS NULL
|
|
|
|
OR (Builds.system = :system));")
|
|
|
|
(stmt (sqlite-prepare db stmt-text #:cache? #t)))
|
|
|
|
(apply sqlite-bind-arguments stmt
|
|
|
|
(query->bind-arguments query))
|
|
|
|
(let ((rows (sqlite-fold-right cons '() stmt)))
|
2020-12-06 11:57:44 +01:00
|
|
|
(sqlite-reset stmt)
|
2019-05-21 11:40:28 +02:00
|
|
|
(and=> (expect-one-row rows) vector->list)))))
|
2019-05-17 22:33:27 +02:00
|
|
|
|
|
|
|
(define (db-get-builds-query-max query)
|
|
|
|
"Return the largest build row identifier matching QUERY."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2019-05-21 11:40:28 +02:00
|
|
|
(let* ((stmt-text "SELECT MAX(Builds.rowid) FROM Builds
|
|
|
|
INNER JOIN Evaluations ON Builds.evaluation = Evaluations.id
|
|
|
|
INNER JOIN Specifications ON Evaluations.specification = Specifications.name
|
|
|
|
WHERE (Builds.nix_name LIKE :query)
|
2020-04-15 19:50:25 +02:00
|
|
|
AND (:status IS NULL
|
|
|
|
OR (Builds.status = :status))
|
2019-05-21 11:40:28 +02:00
|
|
|
AND (:spec IS NULL
|
|
|
|
OR (Specifications.name = :spec))
|
|
|
|
AND (:system IS NULL
|
|
|
|
OR (Builds.system = :system));")
|
|
|
|
(stmt (sqlite-prepare db stmt-text #:cache? #t)))
|
|
|
|
(apply sqlite-bind-arguments stmt
|
|
|
|
(query->bind-arguments query))
|
|
|
|
(let ((rows (sqlite-fold-right cons '() stmt)))
|
2020-12-06 11:57:44 +01:00
|
|
|
(sqlite-reset stmt)
|
2019-05-21 11:40:28 +02:00
|
|
|
(and=> (expect-one-row rows) vector->list)))))
|
2019-05-17 22:33:27 +02:00
|
|
|
|
2018-08-05 16:55:16 +02:00
|
|
|
(define (db-get-builds-min eval status)
|
|
|
|
"Return the min build (stoptime, rowid) pair for the given evaluation EVAL
|
|
|
|
and STATUS."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let ((rows (sqlite-exec db "
|
2020-12-02 17:14:01 +01:00
|
|
|
SELECT stoptime, rowid FROM Builds
|
2018-08-05 16:55:16 +02:00
|
|
|
WHERE evaluation=" eval "
|
|
|
|
AND (" status " IS NULL OR (" status " = 'pending'
|
|
|
|
AND Builds.status < 0)
|
|
|
|
OR (" status " = 'succeeded'
|
|
|
|
AND Builds.status = 0)
|
|
|
|
OR (" status " = 'failed'
|
2020-12-02 17:14:01 +01:00
|
|
|
AND Builds.status > 0))
|
|
|
|
ORDER BY stoptime ASC, rowid ASC
|
|
|
|
LIMIT 1")))
|
2018-09-30 17:26:12 +02:00
|
|
|
(and=> (expect-one-row rows) vector->list))))
|
Add a web interface.
* Makefile.am (dist_sql_DATA): Add static files.
* src/cuirass/database.scm (assqx-ref): Export it.
(db-get-builds): Add 'evaluation' filter and filters for pagination.
(db-get-evaluations-build-summary, db-get-evaluations-id-min,
db-get-evaluations-id-max, db-get-builds-min, db-get-builds-max): New exported
procedures.
* src/cuirass/http.scm (%static-directory): New parameter.
(%page-size, %file-mime-types, %file-white-list): New variables.
(handle-build-request, handle-builds-request): Move the WITH-CRITICAL-SECTION
call out.
(url-handler): Add RESPOND-HTML, RESPOND-STATIC-FILE and RESPOND-NOT-FOUND
procedures. Call WITH-CRITICAL-SECTION sooner for the '/build',
'/api/latestbuilds' and '/api/queue' routes. Add '/', '/jobset/<name>',
'/eval/<id>', '/static/<path>' routes. Use RESPOND-NOT-FOUND when the route
isn't found.
* src/cuirass/templates.scm: New file.
(html-page, specifications-table, evaluation-info-table,
build-eval-table): New exported procedures.
(pagination): New procedure.
Signed-off-by: Clément Lassieur <clement@lassieur.org>
2018-07-21 15:39:10 +02:00
|
|
|
|
2018-08-05 16:55:16 +02:00
|
|
|
(define (db-get-builds-max eval status)
|
|
|
|
"Return the max build (stoptime, rowid) pair for the given evaluation EVAL
|
|
|
|
and STATUS."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
database: Serialize all database accesses in a thread.
Fixes <https://bugs.gnu.org/32234>.
* bin/cuirass.in (main): Keep only one WITH-DATABASE call around all fibers.
Remove all DB arguments.
* src/cuirass/base.scm (evaluate, update-build-statuses!, spawn-builds,
handle-build-event, build-packages): Remove all DB arguments.
(clear-build-queue, cancel-old-builds): Wrap in WITH-DB-CRITICAL-SECTION,
remove all DB arguments.
(restart-builds): Remove the NON-BLOCKING call, remove all DB arguments.
(process-specs): Remove all DB arguments, remove the WITH-DATABASE call.
* src/cuirass/database.scm (%db-channel): New parameter.
(with-db-critical-section): New macro.
(db-add-input, db-add-specification, db-get-inputs, db-get-specifications,
db-add-evaluation, db-add-build, db-update-build-status!, db-get-outputs,
db-get-builds, db-get-build, db-get-pending-derivations, db-get-stamp,
db-add-stamp, db-get-evaluations, db-get-evaluations-build-summary,
db-get-evaluations-id-min, db-get-evaluations-id-max, db-get-builds-min,
db-get-builds-max): Wrap in WITH-DB-CRITICAL-SECTION, remove all DB arguments.
(with-database): Wrap BODY in PARAMETERIZE form that sets %DB-CHANNEL to the
channel returned by MAKE-CRITICAL-SECTION.
* src/cuirass/http.scm (handle-build-request, handle-builds-request): Remove
all DB arguments.
(url-handler): Remove all DB arguments, remove the DB-CHANNEL state, remove
the WITH-CRITICAL-SECTION calls.
(run-cuirass-server): Remove the DB arguments, remove the
MAKE-CRITICAL-SECTION call.
* src/cuirass/utils.scm (make-critical-section): Replace SPAWN-FIBER with
CALL-WITH-NEW-THREAD. Wrap body in PARAMETERIZE form that clears
CURRENT-FIBER.
* tests/database.scm (with-temporary-database, "db-add-specification",
"db-add-build", "db-update-build-status!", "db-get-builds",
"db-get-pending-derivations"): Remove the DB arguments.
("db-init"): Set the %DB-CHANNEL parameter to the channel returned by
MAKE-CRITICAL-SECTION, and return #t.
("database"): Set %DB-CHANNEL to #f during cleanup.
* tests/http.scm ("db-init"): Set the %DB-CHANNEL parameter to the channel
returned by MAKE-CRITICAL-SECTION, and return #t.
("cuirass-run", "fill-db"): Remove the DB arguments.
("http"): Set %DB-CHANNEL to #f during cleanup.
2018-08-05 13:14:44 +02:00
|
|
|
(let ((rows (sqlite-exec db "
|
2020-12-02 17:14:01 +01:00
|
|
|
SELECT stoptime, rowid FROM Builds
|
|
|
|
WHERE evaluation=" eval "
|
2018-08-05 16:55:16 +02:00
|
|
|
AND (" status " IS NULL OR (" status " = 'pending'
|
|
|
|
AND Builds.status < 0)
|
|
|
|
OR (" status " = 'succeeded'
|
|
|
|
AND Builds.status = 0)
|
|
|
|
OR (" status " = 'failed'
|
2020-12-02 17:14:01 +01:00
|
|
|
AND Builds.status > 0))
|
|
|
|
ORDER BY stoptime DESC, rowid DESC
|
|
|
|
LIMIT 1")))
|
2018-09-30 17:26:12 +02:00
|
|
|
(and=> (expect-one-row rows) vector->list))))
|
2018-08-05 21:25:37 +02:00
|
|
|
|
|
|
|
(define (db-get-evaluation-specification eval)
|
|
|
|
"Return specification of evaluation with id EVAL."
|
utils: Change critical section terminology to worker threads.
As far as I'm aware, it's necessary to use a separate thread for interacting
with SQLite as one of the threads used for fibers will be blocked while the
SQLite query is running.
This doesn't mean all queries have to be executed one at a time though,
providing the queries are executed outside the threads used by fibers, and a
single connection isn't used in multiple threads.
These changes start to move in this direction, first by just changing the
terminology.
* src/cuirass/base.scm (clear-build-queue, cancel-old-builds): Change
with-db-critical-section to with-db-worker-thread.
* src/cuirass/database.scm (with-db-critical-section): Rename syntax rule to
with-db-worker-thread.
(db-add-input, db-add-checkout, db-add-specification, db-remove-specification,
db-get-inputs, db-get-specification, db-add-evaluation,
db-set-evaluations-done, db-set-evaluation-done, db-add-derivation-output,
db-add-build, db-update-build-status!, db-get-output, db-get-outputs,
db-get-builds-by-search, db-get-builds, db-get-build derivation-or-id,
db-add-event, db-get-events, db-delete-events-with-ids-<=-to,
db-get-pending-derivations, db-get-checkouts, db-get-evaluations,
db-get-evaluations-build-summary, db-get-evaluations-id-max,
db-get-evaluation-summary, db-get-builds-query-min, db-get-builds-query-max,
db-get-builds-min, db-get-builds-max, db-get-evaluation-specification): Change
from using with-db-critical-section to
with-db-worker-thread.
(with-database): Change syntax rule to use make-worker-thread-channel,
renaming from make-critical-section.
* src/cuirass/utils.scm (%critical-section-args): Rename parameter to
%worker-thread-args.
(make-critical-section): Rename to make-worker-thread-channel, and adjust
parameter and docstring.
(call-with-critical-section): Rename to call-with-worker-thread and adjust
parameter.
(with-critical-section): Rename to with-worker-thread, and adjust to call
call-with-worker-thread.
* tests/database.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
* tests/http.scm (db-init): Use make-worker-thread-channel rather than
make-critical-section.
2020-01-24 19:25:47 +01:00
|
|
|
(with-db-worker-thread db
|
2018-08-05 21:25:37 +02:00
|
|
|
(let ((rows (sqlite-exec db "
|
|
|
|
SELECT specification FROM Evaluations
|
|
|
|
WHERE id = " eval)))
|
2018-09-30 17:26:12 +02:00
|
|
|
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
2020-06-10 15:58:11 +02:00
|
|
|
|
|
|
|
(define (db-get-build-product-path id)
|
|
|
|
"Return the build product with the given ID."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(let ((rows (sqlite-exec db "
|
|
|
|
SELECT path FROM BuildProducts
|
|
|
|
WHERE rowid = " id)))
|
|
|
|
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
|
|
|
|
|
|
|
(define (db-get-build-products build-id)
|
|
|
|
"Return the build products associated to the given BUILD-ID."
|
|
|
|
(with-db-worker-thread db
|
|
|
|
(let loop ((rows (sqlite-exec db "
|
|
|
|
SELECT rowid, type, file_size, checksum, path from BuildProducts
|
|
|
|
WHERE build = " build-id))
|
|
|
|
(products '()))
|
|
|
|
(match rows
|
|
|
|
(() (reverse products))
|
|
|
|
((#(id type file-size checksum path)
|
|
|
|
. rest)
|
|
|
|
(loop rest
|
|
|
|
(cons `((#:id . ,id)
|
|
|
|
(#:type . ,type)
|
|
|
|
(#:file-size . ,file-size)
|
|
|
|
(#:checksum . ,checksum)
|
|
|
|
(#:path . ,path))
|
|
|
|
products)))))))
|