guix
/
data-service
mirror of git://git.savannah.gnu.org/guix/data-service.git
2
0
Fork 0
Code Issues Releases Wiki Activity

Improve deleting derivations 

Drop the batch size to get rid of warnings about memory usage and improve the
logging by adding duration information.
This commit is contained in:
Christopher Baines 2022-07-08 20:54:12 +01:00
parent 788571f53f
commit 39487cd7e6
1 changed files with 35 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
guix-data-service/data-deletion.scm
View File

@ -336,7 +336,7 @@ FROM (
WHERE commit = ''")))))
(define* (delete-unreferenced-derivations #:key
(batch-size 1000000))
(batch-size 100000))
(define (delete-builds-for-derivation-output-details-set
conn
derivation-output-details-set-id)
@ -491,10 +491,11 @@ DELETE FROM derivations WHERE id = $1"
(lambda (conn)
(define (delete-batch conn)
(let* ((derivations
(map car
(exec-query
conn
"
(with-time-logging "fetching batch of derivations"
(map car
(exec-query
conn
"
SELECT DISTINCT derivation_id
FROM derivation_outputs
WHERE NOT EXISTS (
@ -514,36 +515,37 @@ WHERE NOT EXISTS (
SELECT 1 FROM guix_revision_system_test_derivations
WHERE derivation_id = derivation_outputs.derivation_id
) LIMIT $1"
(list (number->string batch-size)))))
(list (number->string batch-size))))))
(derivations-count (length derivations)))
(simple-format (current-error-port)
"Looking at ~A derivations\n"
derivations-count)
(let ((deleted-count
(fold
(lambda (count result)
(+ result count))
0
(map
(lambda (derivation-id)
(unless (string->number derivation-id)
(error
(simple-format #f "derivation-id: ~A is not a number"
derivation-id)))
(with-time-logging
(simple-format #f
"Looking at ~A derivations"
derivations-count)
(fold
(lambda (count result)
(+ result count))
0
(map
(lambda (derivation-id)
(unless (string->number derivation-id)
(error
(simple-format #f "derivation-id: ~A is not a number"
derivation-id)))
(with-thread-postgresql-connection
(lambda (conn)
(with-postgresql-transaction
conn
(lambda (conn)
(exec-query
conn
"
(with-thread-postgresql-connection
(lambda (conn)
(with-postgresql-transaction
conn
(lambda (conn)
(exec-query
conn
"
SET CONSTRAINTS derivations_by_output_details_set_derivation_id_fkey DEFERRED")
(maybe-delete-derivation conn
derivation-id))))))
derivations))))
(maybe-delete-derivation conn
derivation-id))))))
derivations)))))
(simple-format (current-error-port)
"Deleted ~A derivations\n"
deleted-count)
@ -558,10 +560,9 @@ SET CONSTRAINTS derivations_by_output_details_set_derivation_id_fkey DEFERRED")
(let ((batch-deleted-count (delete-batch conn)))
(if (eq? 0 batch-deleted-count)
(begin
(simple-format
(current-output-port)
"Deleting unused derivation_source_files entries")
(delete-unreferenced-derivations-source-files conn)
(with-time-logging
"Deleting unused derivation_source_files entries"
(delete-unreferenced-derivations-source-files conn))
(simple-format
(current-output-port)
"Finished deleting derivations, deleted ~A in total\n"