metrics: Add evaluation related metrics.
* src/cuirass/metrics.scm (db-average-build-start-time-per-eval,
db-average-build-complete-time-per-eval, db-evaluation-completion-speed,
db-latest-evaluations): New procedures.
(%metrics): Add 'average-eval-build-start-time,
'average-eval-build-complete-time, 'evaluation-completion-speed.
(db-update-metrics): Update evaluation related metrics for the evaluations
added the past three days.
* tests/metrics.scm ("average-eval-build-start-time",
"average-eval-build-complete-time", "evaluation-completion-speed"): Add new
tests.
This commit is contained in:
Mathieu Othacehe
parent
822dbb366d
commit
78de0da831
|
|
@ -96,6 +96,58 @@ SELECT 100 * CAST(SUM(status > 0) as float) / COUNT(*) FROM
|
|||
" ORDER BY rowid DESC LIMIT " (or limit -1) ");")))
|
||||
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
||||
|
||||
(define* (db-average-build-start-time-per-eval eval)
|
||||
"Return the average build start time for the given EVAL."
|
||||
(with-db-worker-thread db
|
||||
(let ((rows (sqlite-exec db "\
|
||||
SELECT AVG(B.starttime - E.evaltime) FROM
|
||||
(SELECT id, evaltime
|
||||
FROM Evaluations WHERE id = " eval ") E
|
||||
LEFT JOIN
|
||||
(SELECT id, evaluation, starttime FROM Builds) B
|
||||
ON E.id = B.evaluation and B.starttime > 0
|
||||
GROUP BY E.id;")))
|
||||
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
||||
|
||||
(define* (db-average-build-complete-time-per-eval eval)
|
||||
"Return the average build complete time for the given EVAL."
|
||||
(with-db-worker-thread db
|
||||
(let ((rows (sqlite-exec db "\
|
||||
SELECT AVG(B.stoptime - E.evaltime) FROM
|
||||
(SELECT id, evaltime
|
||||
FROM Evaluations WHERE id = " eval ") E
|
||||
LEFT JOIN
|
||||
(SELECT id, evaluation, stoptime FROM Builds) B
|
||||
ON E.id = B.evaluation and B.stoptime > 0
|
||||
GROUP BY E.id;")))
|
||||
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
||||
|
||||
(define* (db-evaluation-completion-speed eval)
|
||||
"Return the evaluation completion speed of the given EVAL. The speed is
|
||||
expressed in builds per minute."
|
||||
;; completion_speed = 60 * completed_builds / eval_duration.
|
||||
;;
|
||||
;; evaluation_duration (seconds) = current_time - eval_start_time
|
||||
;; If some evaluations builds are not completed.
|
||||
;;
|
||||
;; evaluation_duration (seconds) = max(build_stop_time) - eval_start_time
|
||||
;; If the evaluation builds are all completed.
|
||||
(with-db-worker-thread db
|
||||
(let ((rows (sqlite-exec db "\
|
||||
SELECT
|
||||
60.0 * SUM(B.status = 0) /
|
||||
(CASE SUM(status < 0)
|
||||
WHEN 0 THEN MAX(stoptime)
|
||||
ELSE strftime('%s', 'now')
|
||||
END - E.evaltime) FROM
|
||||
(SELECT id, evaltime
|
||||
FROM Evaluations WHERE id = " eval ") E
|
||||
LEFT JOIN
|
||||
(SELECT id, evaluation, status, stoptime FROM Builds) B
|
||||
ON E.id = B.evaluation and B.stoptime > 0
|
||||
GROUP BY E.id;")))
|
||||
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
||||
|
||||
(define (db-previous-day-timestamp)
|
||||
"Return the timestamp of the previous day."
|
||||
(with-db-worker-thread db
|
||||
|
|
@ -110,6 +162,20 @@ date('now', '-1 day'));")))
|
|||
date('now'));")))
|
||||
(and=> (expect-one-row rows) (cut vector-ref <> 0)))))
|
||||
|
||||
(define* (db-latest-evaluations #:key (days 3))
|
||||
"Return the successful evaluations added during the previous DAYS."
|
||||
(with-db-worker-thread db
|
||||
(let ((query (format #f "SELECT id from Evaluations
|
||||
WHERE date(timestamp, 'unixepoch') > date('now', '-~a day') AND
|
||||
status = 0 ORDER BY rowid DESC" days)))
|
||||
(let loop ((rows (sqlite-exec db query))
|
||||
(evaluations '()))
|
||||
(match rows
|
||||
(() (reverse evaluations))
|
||||
((#(id) . rest)
|
||||
(loop rest
|
||||
(cons id evaluations))))))))
|
||||
|
||||
|
||||
;;;
|
||||
;;; Definitions.
|
||||
|
|
@ -165,7 +231,22 @@ date('now'));")))
|
|||
|
||||
(metric
|
||||
(id 'percentage-failed-eval-per-spec)
|
||||
(compute-proc db-percentage-failed-eval-per-spec))))
|
||||
(compute-proc db-percentage-failed-eval-per-spec))
|
||||
|
||||
;; Average time to start a build for an evaluation.
|
||||
(metric
|
||||
(id 'average-eval-build-start-time)
|
||||
(compute-proc db-average-build-start-time-per-eval))
|
||||
|
||||
;; Average time to complete a build for an evaluation.
|
||||
(metric
|
||||
(id 'average-eval-build-complete-time)
|
||||
(compute-proc db-average-build-complete-time-per-eval))
|
||||
|
||||
;; Evaluation completion speed in builds/minute.
|
||||
(metric
|
||||
(id 'evaluation-completion-speed)
|
||||
(compute-proc db-evaluation-completion-speed))))
|
||||
|
||||
(define (metric->type metric)
|
||||
"Return the index of the given METRIC in %metrics list. This index is used
|
||||
|
|
@ -251,6 +332,11 @@ timestamp) VALUES ("
|
|||
(define specifications
|
||||
(map (cut assq-ref <> #:name) (db-get-specifications)))
|
||||
|
||||
;; We can not update all evaluations metrics for performance reasons. Limit
|
||||
;; to the evaluations that were added during the past three days.
|
||||
(define evaluations
|
||||
(db-latest-evaluations))
|
||||
|
||||
(db-update-metric 'builds-per-day)
|
||||
(db-update-metric 'new-derivations-per-day)
|
||||
(db-update-metric 'pending-builds)
|
||||
|
|
@ -270,4 +356,14 @@ timestamp) VALUES ("
|
|||
'percentage-failure-100-last-eval-per-spec spec)
|
||||
(db-update-metric
|
||||
'percentage-failed-eval-per-spec spec))
|
||||
specifications))
|
||||
specifications)
|
||||
|
||||
;; Update evaluation related metrics.
|
||||
(for-each (lambda (evaluation)
|
||||
(db-update-metric
|
||||
'average-eval-build-start-time evaluation)
|
||||
(db-update-metric
|
||||
'average-eval-build-complete-time evaluation)
|
||||
(db-update-metric
|
||||
'evaluation-completion-speed evaluation))
|
||||
evaluations))
|
||||
|
|
|
|||
|
|
@ -60,36 +60,41 @@
|
|||
(sqlite-exec (%db) "\
|
||||
INSERT INTO Evaluations (specification, status,
|
||||
timestamp, checkouttime, evaltime) VALUES ('guix', -1, 1600174547, 0, 0);")
|
||||
(sqlite-exec (%db) "\
|
||||
(sqlite-exec (%db) (format #f "\
|
||||
INSERT INTO Evaluations (specification, status,
|
||||
timestamp, checkouttime, evaltime) VALUES ('guix', 0, 1600174547, 1600174548,
|
||||
1600260947);")
|
||||
timestamp, checkouttime, evaltime) VALUES ('guix', 0, ~a, ~a, ~a);\
|
||||
" yesterday (+ yesterday 100) (+ yesterday 600)))
|
||||
(sqlite-exec (%db) "\
|
||||
INSERT INTO Evaluations (specification, status,
|
||||
timestamp, checkouttime, evaltime) VALUES ('guix', 1, 1600174547,
|
||||
1600174548, 0);")
|
||||
(sqlite-exec (%db) "\
|
||||
INSERT INTO Evaluations (specification, status,
|
||||
timestamp, checkouttime, evaltime) VALUES ('guix', 2, 1600174547,
|
||||
1600174548, 0);")
|
||||
timestamp, checkouttime, evaltime) VALUES ('guix', 1, 1600174547,
|
||||
1600174548, 1600174647);")
|
||||
(sqlite-exec (%db) (format #f "\
|
||||
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
|
||||
nix_name, log, status, timestamp, starttime, stoptime) VALUES
|
||||
(1, '/gnu/store/xxx.drv', 1, '', '', '', '', 0, ~a, ~a, ~a);\
|
||||
" yesterday (+ yesterday 500) (+ yesterday 1500)))
|
||||
(1, '/gnu/store/1.drv', 2, '', '', '', '', 0, ~a, ~a, ~a);\
|
||||
" yesterday (+ yesterday 1600) (+ yesterday 2600)))
|
||||
(sqlite-exec (%db) (format #f "\
|
||||
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
|
||||
nix_name, log, status, timestamp, starttime, stoptime) VALUES
|
||||
(2, '/gnu/store/yyy.drv', 1, '', '', '', '', -2, 0, 0, 0);"))))
|
||||
(2, '/gnu/store/2.drv', 2, '', '', '', '', -2, 0, 0, 0);"))
|
||||
(sqlite-exec (%db) (format #f "\
|
||||
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
|
||||
nix_name, log, status, timestamp, starttime, stoptime) VALUES
|
||||
(3, '/gnu/store/3.drv', 4, '', '', '', '', 0, 1600174451, 1600174451,
|
||||
1600174651);"))))
|
||||
|
||||
(test-equal "average-eval-duration-per-spec"
|
||||
`(("guix" . 86400.0))
|
||||
`(("guix" . 350.0))
|
||||
(begin
|
||||
(db-update-metric 'average-eval-duration-per-spec "guix")
|
||||
(db-get-metrics-with-id 'average-eval-duration-per-spec)))
|
||||
|
||||
(test-equal "builds-per-day"
|
||||
1.0
|
||||
2.0
|
||||
(begin
|
||||
(db-update-metric 'builds-per-day)
|
||||
(db-get-metric 'builds-per-day yesterday)))
|
||||
|
|
@ -101,7 +106,7 @@ nix_name, log, status, timestamp, starttime, stoptime) VALUES
|
|||
(db-get-metrics-with-id 'pending-builds)))
|
||||
|
||||
(test-equal "new-derivations-per-day"
|
||||
`((,yesterday . 1.0))
|
||||
`((,yesterday . 2.0))
|
||||
(begin
|
||||
(db-update-metric 'new-derivations-per-day)
|
||||
(db-get-metrics-with-id 'new-derivations-per-day)))
|
||||
|
|
@ -118,10 +123,28 @@ nix_name, log, status, timestamp, starttime, stoptime) VALUES
|
|||
(sqlite-exec (%db) (format #f "\
|
||||
INSERT INTO Builds (id, derivation, evaluation, job_name, system,
|
||||
nix_name, log, status, timestamp, starttime, stoptime) VALUES
|
||||
(3, '/gnu/store/zzz.drv', 1, '', '', '', '', -2, 0, 0, 0);"))
|
||||
(4, '/gnu/store/4.drv', 1, '', '', '', '', -2, 0, 0, 0);"))
|
||||
(db-update-metrics)
|
||||
(db-get-metrics-with-id 'pending-builds)))
|
||||
|
||||
(test-equal "average-eval-build-start-time"
|
||||
`((2 . 1000.0))
|
||||
(begin
|
||||
(db-update-metric 'average-eval-build-start-time 2)
|
||||
(db-get-metrics-with-id 'average-eval-build-start-time)))
|
||||
|
||||
(test-equal "average-eval-build-complete-time"
|
||||
`((2 . 2000.0))
|
||||
(begin
|
||||
(db-update-metric 'average-eval-build-complete-time 2)
|
||||
(db-get-metrics-with-id 'average-eval-build-complete-time)))
|
||||
|
||||
(test-equal "evaluation-completion-speed"
|
||||
15.0
|
||||
(begin
|
||||
(db-update-metric 'evaluation-completion-speed 4)
|
||||
(db-get-metric 'evaluation-completion-speed 4)))
|
||||
|
||||
(test-assert "db-close"
|
||||
(db-close (%db)))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue