guix
/
data-service
mirror of git://git.savannah.gnu.org/guix/data-service.git
2
0
Fork 0
Code Issues Releases Wiki Activity

Process derivations in chunks 

Which should reduce the peak memory usage.
This commit is contained in:
Christopher Baines 2022-01-14 15:25:53 +00:00
parent 5a1b6d41eb
commit a7c9daab6a
1 changed files with 26 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
guix-data-service/model/derivation.scm
View File

@ -1801,20 +1801,34 @@ INNER JOIN derivation_source_files
derivation-ids-hash-table derivation-ids-hash-table
derivation-file-names) derivation-file-names)
(let ((missing-derivations (let ((missing-derivation-filenames
(with-time-logging "reading missing derivations" (deduplicate-strings
(map read-derivation-from-file (filter (lambda (derivation-file-name)
(deduplicate-strings (not (hash-ref derivation-ids-hash-table
(filter (lambda (derivation-file-name) derivation-file-name)))
(not (hash-ref derivation-ids-hash-table derivation-file-names))))
derivation-file-name)))
derivation-file-names))))))
(unless (null? missing-derivations) (chunk-for-each!
(insert-missing-derivations conn (lambda (missing-derivation-filenames-chunk)
derivation-ids-hash-table (let ((missing-derivations-chunk
missing-derivations)) (with-time-logging
(simple-format #f "reading ~A missing derivations"
(length missing-derivation-filenames-chunk))
(map read-derivation-from-file
;; Do the filter again, since processing the last
;; chunk might have inserted some of the
;; derivations in this chunk
(filter (lambda (derivation-file-name)
(not (hash-ref derivation-ids-hash-table
derivation-file-name)))
missing-derivation-filenames-chunk)))))
(unless (null? missing-derivations-chunk)
(insert-missing-derivations conn
derivation-ids-hash-table
missing-derivations-chunk))))
1000
missing-derivation-filenames)
(let ((all-ids (let ((all-ids
(map (lambda (derivation-file-name) (map (lambda (derivation-file-name)