From f1989e8758b89b2750f7a546871c0721ab9f9839 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sat, 8 Feb 2020 11:16:08 +0000 Subject: [PATCH] Rework cross derivations support Stop using the system values as targets, and remove package derivation entries where this is the case. Switch the non-cross derivation case to have a target of "", rather than matching the system, as this makes more sense, and is more consistent now that the target values no longer match the system values. Hardcode some more correct target values, and use these instead. Hopefully this can be better integrated with Guix in the future. This commit also includes a migration attempting to shrink some indexes. --- .../jobs/load-new-guix-revision.scm | 51 ++++++++++++++----- guix-data-service/model/derivation.scm | 9 ++++ .../web/repository/controller.scm | 3 +- guix-data-service/web/repository/html.scm | 3 +- guix-data-service/web/revision/controller.scm | 10 ++-- guix-data-service/web/revision/html.scm | 6 ++- ...ivations_by_guix_revision_range_target.sql | 15 ++++++ .../change_package_derivations_target.sql | 7 +++ .../increase_fillfactor_for_some_indexes.sql | 29 +++++++++++ .../deploy/remove_old_cross_derivations.sql | 30 +++++++++++ ...ivations_by_guix_revision_range_target.sql | 7 +++ .../change_package_derivations_target.sql | 7 +++ .../increase_fillfactor_for_some_indexes.sql | 7 +++ .../revert/remove_old_cross_derivations.sql | 7 +++ sqitch/sqitch.plan | 4 ++ ...ivations_by_guix_revision_range_target.sql | 7 +++ .../change_package_derivations_target.sql | 7 +++ .../increase_fillfactor_for_some_indexes.sql | 7 +++ .../verify/remove_old_cross_derivations.sql | 7 +++ 19 files changed, 202 insertions(+), 21 deletions(-) create mode 100644 sqitch/deploy/change_package_derivations_by_guix_revision_range_target.sql create mode 100644 sqitch/deploy/change_package_derivations_target.sql create mode 100644 sqitch/deploy/increase_fillfactor_for_some_indexes.sql create mode 100644 sqitch/deploy/remove_old_cross_derivations.sql create mode 100644 sqitch/revert/change_package_derivations_by_guix_revision_range_target.sql create mode 100644 sqitch/revert/change_package_derivations_target.sql create mode 100644 sqitch/revert/increase_fillfactor_for_some_indexes.sql create mode 100644 sqitch/revert/remove_old_cross_derivations.sql create mode 100644 sqitch/verify/change_package_derivations_by_guix_revision_range_target.sql create mode 100644 sqitch/verify/change_package_derivations_target.sql create mode 100644 sqitch/verify/increase_fillfactor_for_some_indexes.sql create mode 100644 sqitch/verify/remove_old_cross_derivations.sql diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index cc927d8..e494485 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -384,21 +384,37 @@ WHERE job_id = $1" (define inferior-%supported-systems (inferior-eval '(@ (guix packages) %supported-systems) inf)) + (define cross-derivations + `(("x86_64-linux" . ("arm-linux-gnueabihf" + "aarch64-linux-gnu" + "powerpc-linux-gnu" + "riscv64-linux-gnu" + "i586-pc-gnu")))) + (define supported-system-pairs (map (lambda (system) - (cons system system)) + (cons system #f)) inferior-%supported-systems)) (define supported-system-cross-build-pairs - (map (lambda (system) - (filter-map (lambda (target) - (and (not (string=? system target)) - (cons system target))) - inferior-%supported-systems)) - inferior-%supported-systems)) + (append-map + (match-lambda + ((system . targets) + (list + (map (lambda (target) + (cons system target)) + targets)))) + cross-derivations)) (define (proc packages system-target-pairs) `(lambda (store) + (define target-system-alist + '(("arm-linux-gnueabihf" . "armhf-linux") + ("aarch64-linux-gnu" . "aarch64-linux") + ("powerpc-linux-gnu" . "") ; TODO I don't know? + ("riscv64-linux-gnu" . "") ; TODO I don't know? + ("i586-pc-gnu" . "i586-gnu"))) + (define package-transitive-supported-systems-supports-multiple-arguments? #t) (define (get-supported-systems package system) @@ -441,11 +457,11 @@ WHERE job_id = $1" target (let ((file-name (derivation-file-name - (if (string=? system target) - (package-derivation store package system) + (if target (package-cross-derivation store package target - system))))) + system) + (package-derivation store package system))))) (add-temp-root store file-name) file-name)))) (lambda args @@ -470,9 +486,16 @@ WHERE job_id = $1" package system target)) - (lset-intersection - string=? - supported-systems + (filter + (match-lambda + (#f #t) ; No target + (target + (let ((system-for-target + (assoc-ref target-system-alist + target))) + (member system-for-target + supported-systems + string=?)))) (list ,@(map cdr system-target-pairs)))) '()))) (delete-duplicates @@ -639,7 +662,7 @@ WHERE job_id = $1" (list (inferior-package-id->package-database-id inferior-package-id) system - target))) + (or target "")))) inferior-data-4-tuples))) diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index 033168b..de675d5 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -34,6 +34,7 @@ #:use-module (guix-data-service database) #:use-module (guix-data-service model utils) #:export (valid-systems + valid-targets count-derivations select-derivation-by-file-name select-derivation-by-file-name-hash @@ -66,6 +67,14 @@ "mips64el-linux" "x86_64-linux")) +(define (valid-targets conn) + '("" ;; no target + "arm-linux-gnueabihf" + "aarch64-linux-gnu" + "powerpc-linux-gnu" + "riscv64-linux-gnu" + "i586-pc-gnu")) + (define (count-derivations conn) (first (exec-query diff --git a/guix-data-service/web/repository/controller.scm b/guix-data-service/web/repository/controller.scm index aaf763c..1fe4c8c 100644 --- a/guix-data-service/web/repository/controller.scm +++ b/guix-data-service/web/repository/controller.scm @@ -257,7 +257,7 @@ `((system ,(parse-build-system conn) #:default "x86_64-linux") (target ,(parse-build-system conn) - #:default "x86_64-linux"))))) + #:default ""))))) (let* ((system (assq-ref parsed-query-parameters 'system)) (target @@ -304,5 +304,6 @@ branch-name package-name (valid-systems conn) + (valid-targets conn) build-server-urls package-derivations))))))) diff --git a/guix-data-service/web/repository/html.scm b/guix-data-service/web/repository/html.scm index b001be7..8f1a6d5 100644 --- a/guix-data-service/web/repository/html.scm +++ b/guix-data-service/web/repository/html.scm @@ -327,6 +327,7 @@ branch-name package-name valid-systems + valid-targets build-server-urls derivations-by-revision-range) (define versions-list @@ -383,7 +384,7 @@ #:help-text "Show derivations with this system.") ,(form-horizontal-control "Target" query-parameters - #:options valid-systems + #:options valid-targets #:allow-selecting-multiple-options #f #:help-text "Show derivations with this target.") (div (@ (class "form-group form-group-lg")) diff --git a/guix-data-service/web/revision/controller.scm b/guix-data-service/web/revision/controller.scm index 53ec03a..e1b1460 100644 --- a/guix-data-service/web/revision/controller.scm +++ b/guix-data-service/web/revision/controller.scm @@ -172,7 +172,7 @@ request `((search_query ,identity) (system ,parse-system #:multi-value) - (target ,parse-system #:multi-value) + (target ,identity #:multi-value) (maximum_builds ,parse-number) (minimum_builds ,parse-number) (after_name ,identity) @@ -201,7 +201,7 @@ (output_consistency ,identity #:default "any") (system ,parse-system #:default "x86_64-linux") - (target ,parse-system #:default "x86_64-linux") + (target ,identity) (limit_results ,parse-result-limit #:no-default-when (all_results) #:default 10) @@ -242,7 +242,7 @@ `((build_status ,parse-build-status #:multi-value) (build_server ,(parse-build-server conn) #:multi-value) (system ,parse-system #:default "x86_64-linux") - (target ,parse-system #:default "x86_64-linux"))))) + (target ,identity))))) (render-revision-builds mime-types conn @@ -661,6 +661,7 @@ #:sxml (view-revision-derivations commit-hash query-parameters (valid-systems conn) + (valid-targets conn) '() '() #f @@ -717,6 +718,7 @@ #:sxml (view-revision-derivations commit-hash query-parameters (valid-systems conn) + (valid-targets conn) derivations build-server-urls show-next-page? @@ -748,6 +750,7 @@ '() '() '() + '() #:path-base path-base #:header-text header-text #:header-link header-link)))) @@ -790,6 +793,7 @@ derivation-outputs build-server-urls (valid-systems conn) + (valid-targets conn) show-next-page? #:path-base path-base #:header-text header-text diff --git a/guix-data-service/web/revision/html.scm b/guix-data-service/web/revision/html.scm index ce8fc76..36cd510 100644 --- a/guix-data-service/web/revision/html.scm +++ b/guix-data-service/web/revision/html.scm @@ -967,6 +967,7 @@ figure { (define* (view-revision-derivations commit-hash query-parameters valid-systems + valid-targets derivations build-server-urls show-next-page? @@ -1007,7 +1008,7 @@ figure { #:font-family "monospace") ,(form-horizontal-control "Target" query-parameters - #:options valid-systems + #:options valid-targets #:help-text "Only include derivations that are build for this system." #:font-family "monospace") ,(form-horizontal-control @@ -1096,6 +1097,7 @@ figure { derivation-outputs build-server-urls valid-systems + valid-targets show-next-page? #:key (path-base "/revision/") header-text @@ -1144,7 +1146,7 @@ figure { #:font-family "monospace") ,(form-horizontal-control "Target" query-parameters - #:options valid-systems + #:options valid-targets #:allow-selecting-multiple-options #f #:help-text "Only include outputs from derivations that are build for this system." #:font-family "monospace") diff --git a/sqitch/deploy/change_package_derivations_by_guix_revision_range_target.sql b/sqitch/deploy/change_package_derivations_by_guix_revision_range_target.sql new file mode 100644 index 0000000..7ba90c7 --- /dev/null +++ b/sqitch/deploy/change_package_derivations_by_guix_revision_range_target.sql @@ -0,0 +1,15 @@ +-- Deploy guix-data-service:change_package_derivations_by_guix_revision_range_target to pg + +BEGIN; + +UPDATE package_derivations_by_guix_revision_range SET target = '' WHERE system = target; + +DELETE FROM package_derivations_by_guix_revision_range WHERE target IN ( + 'aarch64-linux', + 'armhf-linux', + 'i686-linux', + 'mips64el-linux', + 'x86_64-linux' +); + +COMMIT; diff --git a/sqitch/deploy/change_package_derivations_target.sql b/sqitch/deploy/change_package_derivations_target.sql new file mode 100644 index 0000000..9a9f3c3 --- /dev/null +++ b/sqitch/deploy/change_package_derivations_target.sql @@ -0,0 +1,7 @@ +-- Deploy guix-data-service:change_package_derivations_target to pg + +BEGIN; + +UPDATE package_derivations SET target = '' WHERE system = target; + +COMMIT; diff --git a/sqitch/deploy/increase_fillfactor_for_some_indexes.sql b/sqitch/deploy/increase_fillfactor_for_some_indexes.sql new file mode 100644 index 0000000..93f8209 --- /dev/null +++ b/sqitch/deploy/increase_fillfactor_for_some_indexes.sql @@ -0,0 +1,29 @@ +-- Deploy guix-data-service:increase_fillfactor_for_some_indexes to pg + +BEGIN; + +ALTER INDEX derivation_inputs_pkey SET (fillfactor = 100); +ALTER INDEX derivation_inputs_derivation_output_id_idx SET (fillfactor = 100); +ALTER INDEX guix_revision_package_derivations_pkey SET (fillfactor = 100); +ALTER INDEX derivations_pkey SET (fillfactor = 100); +ALTER INDEX file_name_unique SET (fillfactor = 100); +ALTER INDEX id_unique SET (fillfactor = 100); +ALTER INDEX derivations_hash SET (fillfactor = 100); +ALTER INDEX package_derivations_pkey SET (fillfactor = 100); +ALTER INDEX package_derivations_id_key SET (fillfactor = 100); +ALTER INDEX package_derivations_derivation_id SET (fillfactor = 100); +ALTER INDEX package_derivations_package_id_idx SET (fillfactor = 100); + +REINDEX INDEX derivation_inputs_pkey; +REINDEX INDEX derivation_inputs_derivation_output_id_idx; +REINDEX INDEX guix_revision_package_derivations_pkey; +REINDEX INDEX derivations_pkey; +REINDEX INDEX file_name_unique; +REINDEX INDEX id_unique; +REINDEX INDEX derivations_hash; +REINDEX INDEX package_derivations_pkey; +REINDEX INDEX package_derivations_id_key; +REINDEX INDEX package_derivations_derivation_id; +REINDEX INDEX package_derivations_package_id_idx; + +COMMIT; diff --git a/sqitch/deploy/remove_old_cross_derivations.sql b/sqitch/deploy/remove_old_cross_derivations.sql new file mode 100644 index 0000000..b035673 --- /dev/null +++ b/sqitch/deploy/remove_old_cross_derivations.sql @@ -0,0 +1,30 @@ +-- Deploy guix-data-service:remove_old_cross_derivations to pg + +BEGIN; + +DELETE FROM guix_revision_package_derivations WHERE package_derivation_id IN ( + SELECT id + FROM package_derivations + WHERE target IN ( + 'aarch64-linux', + 'armhf-linux', + 'i686-linux', + 'mips64el-linux', + 'x86_64-linux' + ) +); + +-- Checking this constraint is expensive, so drop it, then re-create it +ALTER TABLE guix_revision_package_derivations DROP CONSTRAINT guix_revision_package_derivations_package_derivation_id_fkey; + +DELETE FROM package_derivations WHERE target IN ( + 'aarch64-linux', + 'armhf-linux', + 'i686-linux', + 'mips64el-linux', + 'x86_64-linux' +); + +ALTER TABLE guix_revision_package_derivations ADD CONSTRAINT "guix_revision_package_derivations_package_derivation_id_fkey" FOREIGN KEY (package_derivation_id) REFERENCES package_derivations(id); + +COMMIT; diff --git a/sqitch/revert/change_package_derivations_by_guix_revision_range_target.sql b/sqitch/revert/change_package_derivations_by_guix_revision_range_target.sql new file mode 100644 index 0000000..3742e92 --- /dev/null +++ b/sqitch/revert/change_package_derivations_by_guix_revision_range_target.sql @@ -0,0 +1,7 @@ +-- Revert guix-data-service:change_package_derivations_by_guix_revision_range_target from pg + +BEGIN; + +-- XXX Add DDLs here. + +COMMIT; diff --git a/sqitch/revert/change_package_derivations_target.sql b/sqitch/revert/change_package_derivations_target.sql new file mode 100644 index 0000000..febb75b --- /dev/null +++ b/sqitch/revert/change_package_derivations_target.sql @@ -0,0 +1,7 @@ +-- Revert guix-data-service:change_package_derivations_target from pg + +BEGIN; + +-- XXX Add DDLs here. + +COMMIT; diff --git a/sqitch/revert/increase_fillfactor_for_some_indexes.sql b/sqitch/revert/increase_fillfactor_for_some_indexes.sql new file mode 100644 index 0000000..a87f3f3 --- /dev/null +++ b/sqitch/revert/increase_fillfactor_for_some_indexes.sql @@ -0,0 +1,7 @@ +-- Revert guix-data-service:increase_fillfactor_for_some_indexes from pg + +BEGIN; + +-- XXX Add DDLs here. + +COMMIT; diff --git a/sqitch/revert/remove_old_cross_derivations.sql b/sqitch/revert/remove_old_cross_derivations.sql new file mode 100644 index 0000000..7577a08 --- /dev/null +++ b/sqitch/revert/remove_old_cross_derivations.sql @@ -0,0 +1,7 @@ +-- Revert guix-data-service:remove_old_cross_derivations from pg + +BEGIN; + +-- XXX Add DDLs here. + +COMMIT; diff --git a/sqitch/sqitch.plan b/sqitch/sqitch.plan index b4e99fd..2d04104 100644 --- a/sqitch/sqitch.plan +++ b/sqitch/sqitch.plan @@ -44,3 +44,7 @@ derivation_source_files_store_path_hash_index 2019-12-29T17:53:08Z Christopher B build_servers_build_config 2020-01-05T12:06:13Z Christopher Baines # Add build_servers_build_config table git-repositories-x-git-repo-header 2020-01-11T16:39:32Z Christopher Baines # Add x_git_repo_header to git_repositories system_test_tables 2020-02-02T11:36:20Z Christopher Baines # Add tables for storing system tests +change_package_derivations_target 2020-02-07T19:29:50Z Christopher Baines # Change package_derivations target +remove_old_cross_derivations 2020-02-07T19:42:54Z Christopher Baines # Remove old cross derivations +increase_fillfactor_for_some_indexes 2020-02-07T20:49:17Z Christopher Baines # Increase the fillfactor for some btree indexes +change_package_derivations_by_guix_revision_range_target 2020-02-08T10:13:07Z Christopher Baines # Change the values for package_derivations_by_guix_revision_range target diff --git a/sqitch/verify/change_package_derivations_by_guix_revision_range_target.sql b/sqitch/verify/change_package_derivations_by_guix_revision_range_target.sql new file mode 100644 index 0000000..c6a756f --- /dev/null +++ b/sqitch/verify/change_package_derivations_by_guix_revision_range_target.sql @@ -0,0 +1,7 @@ +-- Verify guix-data-service:change_package_derivations_by_guix_revision_range_target on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK; diff --git a/sqitch/verify/change_package_derivations_target.sql b/sqitch/verify/change_package_derivations_target.sql new file mode 100644 index 0000000..de6cf7a --- /dev/null +++ b/sqitch/verify/change_package_derivations_target.sql @@ -0,0 +1,7 @@ +-- Verify guix-data-service:change_package_derivations_target on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK; diff --git a/sqitch/verify/increase_fillfactor_for_some_indexes.sql b/sqitch/verify/increase_fillfactor_for_some_indexes.sql new file mode 100644 index 0000000..44269dc --- /dev/null +++ b/sqitch/verify/increase_fillfactor_for_some_indexes.sql @@ -0,0 +1,7 @@ +-- Verify guix-data-service:increase_fillfactor_for_some_indexes on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK; diff --git a/sqitch/verify/remove_old_cross_derivations.sql b/sqitch/verify/remove_old_cross_derivations.sql new file mode 100644 index 0000000..8f6768e --- /dev/null +++ b/sqitch/verify/remove_old_cross_derivations.sql @@ -0,0 +1,7 @@ +-- Verify guix-data-service:remove_old_cross_derivations on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK;