Use much smaller chunks when trying to run the derivation linter
Since larger chunks still ran in to inferior memory usage problems.
This commit is contained in:
parent
aa8c9dbffa
commit
b3d59c650a
|
@ -457,6 +457,50 @@ WHERE job_id = $1")
|
||||||
"vi_VN.UTF-8"
|
"vi_VN.UTF-8"
|
||||||
"zh_CN.UTF-8"))
|
"zh_CN.UTF-8"))
|
||||||
|
|
||||||
|
(define (cleanup-inferior inf)
|
||||||
|
(format (current-error-port)
|
||||||
|
"inferior heap before cleanup: ~a MiB used (~a MiB heap)~%"
|
||||||
|
(round
|
||||||
|
(/ (inferior-eval
|
||||||
|
'(let ((stats (gc-stats)))
|
||||||
|
(- (assoc-ref stats 'heap-size)
|
||||||
|
(assoc-ref stats 'heap-free-size)))
|
||||||
|
inf)
|
||||||
|
(expt 2. 20)))
|
||||||
|
(round
|
||||||
|
(/ (inferior-eval '(assoc-ref (gc-stats) 'heap-size) inf)
|
||||||
|
(expt 2. 20))))
|
||||||
|
|
||||||
|
;; Clean the cached store connections, as there are caches associated with
|
||||||
|
;; these that take up lots of memory
|
||||||
|
(inferior-eval
|
||||||
|
'(when (defined? '%store-table) (hash-clear! %store-table))
|
||||||
|
inf)
|
||||||
|
|
||||||
|
(catch
|
||||||
|
'match-error
|
||||||
|
(lambda ()
|
||||||
|
(inferior-eval '(invalidate-derivation-caches!) inf))
|
||||||
|
(lambda (key . args)
|
||||||
|
(simple-format
|
||||||
|
(current-error-port)
|
||||||
|
"warning: ignoring match-error from calling inferior invalidate-derivation-caches!\n")))
|
||||||
|
|
||||||
|
(inferior-eval '(gc) inf)
|
||||||
|
|
||||||
|
(format (current-error-port)
|
||||||
|
"inferior heap after cleanup: ~a MiB used (~a MiB heap)~%"
|
||||||
|
(round
|
||||||
|
(/ (inferior-eval
|
||||||
|
'(let ((stats (gc-stats)))
|
||||||
|
(- (assoc-ref stats 'heap-size)
|
||||||
|
(assoc-ref stats 'heap-free-size)))
|
||||||
|
inf)
|
||||||
|
(expt 2. 20)))
|
||||||
|
(round
|
||||||
|
(/ (inferior-eval '(assoc-ref (gc-stats) 'heap-size) inf)
|
||||||
|
(expt 2. 20)))))
|
||||||
|
|
||||||
(define (lint-warnings-for-checker packages checker-name)
|
(define (lint-warnings-for-checker packages checker-name)
|
||||||
`(lambda (store)
|
`(lambda (store)
|
||||||
(let* ((checker (find (lambda (checker)
|
(let* ((checker (find (lambda (checker)
|
||||||
|
@ -591,7 +635,7 @@ WHERE job_id = $1")
|
||||||
(with-time-logging (simple-format #f "getting ~A lint warnings"
|
(with-time-logging (simple-format #f "getting ~A lint warnings"
|
||||||
name)
|
name)
|
||||||
(let loop ((packages-chunks
|
(let loop ((packages-chunks
|
||||||
(chunk packages 15000))
|
(chunk packages 4000))
|
||||||
(warnings '()))
|
(warnings '()))
|
||||||
|
|
||||||
(if (null? packages-chunks)
|
(if (null? packages-chunks)
|
||||||
|
@ -600,22 +644,10 @@ WHERE job_id = $1")
|
||||||
(inferior-eval-with-store
|
(inferior-eval-with-store
|
||||||
inf
|
inf
|
||||||
store
|
store
|
||||||
(lint-warnings-for-checker (car packages-chunks) name))))
|
(lint-warnings-for-checker (car packages-chunks)
|
||||||
|
name))))
|
||||||
;; Clean the cached store connections, as there are caches
|
|
||||||
;; associated with these that take up lots of memory
|
|
||||||
(inferior-eval
|
|
||||||
'(when (defined? '%store-table) (hash-clear! %store-table))
|
|
||||||
inf)
|
|
||||||
|
|
||||||
(inferior-eval '(gc) inf)
|
|
||||||
|
|
||||||
(format (current-error-port)
|
|
||||||
"inferior heap size: ~a MiB~%"
|
|
||||||
(round
|
|
||||||
(/ (inferior-eval '(assoc-ref (gc-stats) 'heap-size) inf)
|
|
||||||
(expt 2. 20))))
|
|
||||||
|
|
||||||
|
(cleanup-inferior inf)
|
||||||
(loop (cdr packages-chunks)
|
(loop (cdr packages-chunks)
|
||||||
(append! warnings
|
(append! warnings
|
||||||
new-warnings))))))))))
|
new-warnings))))))))))
|
||||||
|
|
Loading…
Reference in New Issue