From a7c9daab6ace33858be1dcde60f147c3c84afa81 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Fri, 14 Jan 2022 15:25:53 +0000 Subject: [PATCH] Process derivations in chunks Which should reduce the peak memory usage. --- guix-data-service/model/derivation.scm | 38 ++++++++++++++++++-------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index fddf0e3..a1ae821 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -1801,20 +1801,34 @@ INNER JOIN derivation_source_files derivation-ids-hash-table derivation-file-names) - (let ((missing-derivations - (with-time-logging "reading missing derivations" - (map read-derivation-from-file - (deduplicate-strings - (filter (lambda (derivation-file-name) - (not (hash-ref derivation-ids-hash-table - derivation-file-name))) - derivation-file-names)))))) + (let ((missing-derivation-filenames + (deduplicate-strings + (filter (lambda (derivation-file-name) + (not (hash-ref derivation-ids-hash-table + derivation-file-name))) + derivation-file-names)))) - (unless (null? missing-derivations) - (insert-missing-derivations conn - derivation-ids-hash-table - missing-derivations)) + (chunk-for-each! + (lambda (missing-derivation-filenames-chunk) + (let ((missing-derivations-chunk + (with-time-logging + (simple-format #f "reading ~A missing derivations" + (length missing-derivation-filenames-chunk)) + (map read-derivation-from-file + ;; Do the filter again, since processing the last + ;; chunk might have inserted some of the + ;; derivations in this chunk + (filter (lambda (derivation-file-name) + (not (hash-ref derivation-ids-hash-table + derivation-file-name))) + missing-derivation-filenames-chunk))))) + (unless (null? missing-derivations-chunk) + (insert-missing-derivations conn + derivation-ids-hash-table + missing-derivations-chunk)))) + 1000 + missing-derivation-filenames) (let ((all-ids (map (lambda (derivation-file-name)