guix/data-service
2
0
Fork 0
mirror of git://git.savannah.gnu.org/guix/data-service.git synced 2023-12-14 03:23:03 +01:00
Code Issues Releases Wiki Activity

Change the locale codeset representation

Browse source 
From the normalized one, to the one actually contained within glibc. Recent
versions of glibc also contain symlinks linking the normalized codeset to the
locales with the .UTF-8 ending, but older ones do not.

Maybe handling codeset normalisation for queries would be good, but the locale
values ending in .UTF-8 are more compatible and allow the code to be
simplified. For querying, maybe there should be a locales table which handles
different representations.
This commit is contained in:
Christopher Baines 2020-09-26 11:45:57 +01:00
parent af2e12a9ef
commit 53341c70fc
17 changed files with 98 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
guix-data-service/comparison.scm
View file

@ -634,7 +634,7 @@ WITH base_lint_warnings AS (
ORDER BY lint_checkers.id,
CASE
WHEN lint_checker_descriptions.locale = $3 THEN 2
WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_lint_checker_descriptions
@ -653,7 +653,7 @@ WITH base_lint_warnings AS (
ORDER BY lint_warnings.id,
CASE
WHEN lint_warning_messages.locale = $3 THEN 2
WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
ELSE 0 END DESC
), target_lint_warnings AS (
SELECT DISTINCT ON (lint_warnings.id) lint_warnings.id,
@ -684,7 +684,7 @@ WITH base_lint_warnings AS (
ORDER BY lint_checkers.id,
CASE
WHEN lint_checker_descriptions.locale = $3 THEN 2
WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_lint_checker_descriptions
@ -703,7 +703,7 @@ WITH base_lint_warnings AS (
ORDER BY lint_warnings.id,
CASE
WHEN lint_warning_messages.locale = $3 THEN 2
WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
ELSE 0 END DESC
)
SELECT coalesce(

54
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -366,19 +366,19 @@ WHERE job_id = $1"
(define (all-inferior-lint-warnings inf store packages)
(define locales
'("cs_CZ.utf8"
"da_DK.utf8"
"de_DE.utf8"
"eo_EO.utf8"
"es_ES.utf8"
"fr_FR.utf8"
"hu_HU.utf8"
"pl_PL.utf8"
"pt_BR.utf8"
;;"sr_SR.utf8"
"sv_SE.utf8"
"vi_VN.utf8"
"zh_CN.utf8"))
'("cs_CZ.UTF-8"
"da_DK.UTF-8"
"de_DE.UTF-8"
"eo_EO.UTF-8"
"es_ES.UTF-8"
"fr_FR.UTF-8"
"hu_HU.UTF-8"
"pl_PL.UTF-8"
"pt_BR.UTF-8"
;;"sr_SR.UTF-8"
"sv_SE.UTF-8"
"vi_VN.UTF-8"
"zh_CN.UTF-8"))
(define (lint-warnings-for-checker checker-name)
`(lambda (store)
@ -406,7 +406,7 @@ WHERE job_id = $1"
file)
line
column)))
(let* ((source-locale "en_US.utf8")
(let* ((source-locale "en_US.UTF-8")
(source-message
(begin
(setlocale LC_MESSAGES source-locale)
@ -464,7 +464,7 @@ WHERE job_id = $1"
(inferior-eval
`(begin
(define (lint-descriptions-by-locale checker)
(let* ((source-locale "en_US.utf8")
(let* ((source-locale "en_US.UTF-8")
(source-description
(begin
(setlocale LC_MESSAGES source-locale)
@ -1125,24 +1125,14 @@ WHERE job_id = $1"
(error "error: inferior is #f"))
;; Normalise the locale for the inferior process
(catch
#t
(with-exception-handler
(lambda (key . args)
(simple-format
(current-error-port)
"warning: failed to set locale to en_US.UTF-8: ~A ~A\n"
key args))
(lambda ()
(inferior-eval '(setlocale LC_ALL "en_US.utf8") inf))
(lambda (key . args)
(simple-format (current-error-port)
"warning: failed to set locale to en_US.utf8: ~A ~A\n"
key args)
(display "trying to setlocale to en_US.UTF-8 instead\n"
(current-error-port))
(with-exception-handler
(lambda (key . args)
(simple-format
(current-error-port)
"warning: failed to set locale to en_US.UTF-8: ~A ~A\n"
key args))
(lambda ()
(inferior-eval '(setlocale LC_ALL "en_US.UTF-8") inf)))))
(inferior-eval '(setlocale LC_ALL "en_US.UTF-8") inf)))
(inferior-eval '(use-modules (srfi srfi-1)
(srfi srfi-34)

4
guix-data-service/model/lint-checker.scm
View file

@ -93,7 +93,7 @@ INNER JOIN lint_checker_description_sets
ON lint_checkers.lint_checker_description_set_id = lint_checker_description_sets.id
INNER JOIN lint_checker_descriptions
ON lint_checker_descriptions.id = ANY (lint_checker_description_sets.description_ids)
AND lint_checker_descriptions.locale = 'en_US.utf8'
AND lint_checker_descriptions.locale = 'en_US.UTF-8'
INNER JOIN (
SELECT lint_checker_id, COUNT(*)
FROM lint_warnings
@ -138,7 +138,7 @@ INNER JOIN lint_checker_description_sets
ON lint_checkers.lint_checker_description_set_id = lint_checker_description_sets.id
INNER JOIN lint_checker_descriptions
ON lint_checker_descriptions.id = ANY (lint_checker_description_sets.description_ids)
AND lint_checker_descriptions.locale = 'en_US.utf8'
AND lint_checker_descriptions.locale = 'en_US.UTF-8'
WHERE lint_checkers.id IN (
SELECT lint_checker_id
FROM guix_revision_lint_checkers

8
guix-data-service/model/lint-warning.scm
View file

@ -90,7 +90,7 @@ INNER JOIN (
ORDER BY lint_checkers.id,
CASE
WHEN lint_checker_descriptions.locale = $2 THEN 2
WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_lint_checker_descriptions
@ -131,7 +131,7 @@ INNER JOIN lint_warning_messages
" ORDER BY lint_warnings.id,
CASE
WHEN lint_warning_messages.locale = $2 THEN 2
WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC"))
@ -174,7 +174,7 @@ INNER JOIN (
ORDER BY lint_checkers.id,
CASE
WHEN lint_checker_descriptions.locale = $4 THEN 2
WHEN lint_checker_descriptions.locale = 'en_US.utf8' THEN 1
WHEN lint_checker_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_lint_checker_descriptions
@ -202,7 +202,7 @@ WHERE packages.id IN (
ORDER BY lint_warnings.id,
CASE
WHEN lint_warning_messages.locale = $4 THEN 2
WHEN lint_warning_messages.locale = 'en_US.utf8' THEN 1
WHEN lint_warning_messages.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
")

40
guix-data-service/model/package-metadata.scm
View file

@ -42,19 +42,19 @@
description-counts-by-locale))
(define locales
'("cs_CZ.utf8"
"da_DK.utf8"
"de_DE.utf8"
"eo_EO.utf8"
"es_ES.utf8"
"fr_FR.utf8"
"hu_HU.utf8"
"pl_PL.utf8"
"pt_BR.utf8"
;;"sr_SR.utf8"
"sv_SE.utf8"
"vi_VN.utf8"
"zh_CN.utf8"))
'("cs_CZ.UTF-8"
"da_DK.UTF-8"
"de_DE.UTF-8"
"eo_EO.UTF-8"
"es_ES.UTF-8"
"fr_FR.UTF-8"
"hu_HU.UTF-8"
"pl_PL.UTF-8"
"pt_BR.UTF-8"
;;"sr_SR.UTF-8"
"sv_SE.UTF-8"
"vi_VN.UTF-8"
"zh_CN.UTF-8"))
(define inferior-package-id
(@@ (guix inferior) inferior-package-id))
@ -144,7 +144,7 @@ INNER JOIN (
AND packages.version = $3
ORDER BY package_description_sets.id,
CASE WHEN package_descriptions.locale = $4 THEN 2
WHEN package_descriptions.locale = 'en_US.utf8' THEN 1
WHEN package_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_package_descriptions
@ -163,7 +163,7 @@ INNER JOIN (
AND packages.version = $3
ORDER BY package_synopsis_sets.id,
CASE WHEN package_synopsis.locale = $4 THEN 2
WHEN package_synopsis.locale = 'en_US.utf8' THEN 1
WHEN package_synopsis.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_package_synopsis
@ -217,7 +217,7 @@ WHERE packages.id IN (
(define (translate inferior-package-id)
`(let* ((package (hashv-ref %package-table ,inferior-package-id))
(source-locale "en_US.utf8")
(source-locale "en_US.UTF-8")
(source-synopsis
(begin
(setlocale LC_MESSAGES source-locale)
@ -505,7 +505,7 @@ INSERT INTO package_metadata_tsvectors (package_metadata_id, locale,
synopsis_and_description, package_synopsis_id, package_description_id)
SELECT DISTINCT ON (package_metadata.id, locale)
package_metadata.id,
CASE WHEN translated_package_synopsis.locale != 'en_US.utf8'
CASE WHEN translated_package_synopsis.locale != 'en_US.UTF-8'
THEN translated_package_synopsis.locale
ELSE translated_package_descriptions.locale
END AS locale,
@ -524,7 +524,7 @@ INNER JOIN (
INNER JOIN package_descriptions
ON package_descriptions.id = ANY (package_description_sets.description_ids)
ORDER BY package_description_sets.id,
CASE WHEN package_descriptions.locale = 'en_US.utf8' THEN 1
CASE WHEN package_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 2
END DESC
) AS translated_package_descriptions
@ -537,7 +537,7 @@ INNER JOIN (
INNER JOIN package_synopsis
ON package_synopsis.id = ANY (package_synopsis_sets.synopsis_ids)
ORDER BY package_synopsis_sets.id,
CASE WHEN package_synopsis.locale = 'en_US.utf8' THEN 1
CASE WHEN package_synopsis.locale = 'en_US.UTF-8' THEN 1
ELSE 2
END DESC
) AS translated_package_synopsis
@ -545,7 +545,7 @@ INNER JOIN (
translated_package_synopsis.package_synopsis_set_id
AND (translated_package_descriptions.locale =
translated_package_synopsis.locale
OR translated_package_descriptions.locale = 'en_US.utf8')
OR translated_package_descriptions.locale = 'en_US.UTF-8')
WHERE package_metadata.id IN ("
(string-join
(map number->string

6
guix-data-service/model/package.scm
View file

@ -86,7 +86,7 @@ WITH data AS (
ON package_synopsis.id = ANY (package_synopsis_sets.synopsis_ids)
ORDER BY package_synopsis_sets.id,
CASE WHEN package_synopsis.locale = $2 THEN 2
WHEN package_synopsis.locale = 'en_US.utf8' THEN 1
WHEN package_synopsis.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_package_synopsis
@ -99,7 +99,7 @@ WITH data AS (
ON package_descriptions.id = ANY (package_description_sets.description_ids)
ORDER BY package_description_sets.id,
CASE WHEN package_descriptions.locale = $2 THEN 2
WHEN package_descriptions.locale = 'en_US.utf8' THEN 1
WHEN package_descriptions.locale = 'en_US.UTF-8' THEN 1
ELSE 0
END DESC
) AS translated_package_descriptions
@ -184,7 +184,7 @@ WITH search_results AS (
)
AND package_metadata_tsvectors.synopsis_and_description @@ plainto_tsquery($2)
ORDER BY name,
CASE WHEN package_metadata_tsvectors.locale = 'en_US.utf8' THEN 2
CASE WHEN package_metadata_tsvectors.locale = 'en_US.UTF-8' THEN 2
WHEN package_metadata_tsvectors.locale = $3 THEN 1
ELSE 0
END DESC

4
guix-data-service/web/compare/controller.scm
View file

@ -74,7 +74,7 @@
request
`((base_commit ,(parse-commit conn) #:required)
(target_commit ,(parse-commit conn) #:required)
(locale ,identity #:default "en_US.utf8")))))
(locale ,identity #:default "en_US.UTF-8")))))
(render-compare mime-types
conn
parsed-query-parameters)))
@ -86,7 +86,7 @@
(base_datetime ,parse-datetime #:required)
(target_branch ,identity #:required)
(target_datetime ,parse-datetime #:required)
(locale ,identity #:default "en_US.utf8")))))
(locale ,identity #:default "en_US.UTF-8")))))
(render-compare-by-datetime mime-types
conn
parsed-query-parameters)))

6
guix-data-service/web/repository/controller.scm
View file

@ -172,7 +172,7 @@
(guard-against-mutually-exclusive-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.utf8")
`((locale ,identity #:default "en_US.UTF-8")
(after_name ,identity)
(field ,identity #:multi-value
#:default ("version" "synopsis"))
@ -262,7 +262,7 @@
(let ((parsed-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.utf8")
`((locale ,identity #:default "en_US.UTF-8")
(package_query ,identity)
(linter ,identity #:multi-value)
(message_query ,identity)
@ -293,7 +293,7 @@
(parsed-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.utf8")))))
`((locale ,identity #:default "en_US.UTF-8")))))
(if commit-hash
(render-revision-package-version mime-types
conn

6
guix-data-service/web/revision/controller.scm
View file

@ -125,7 +125,7 @@
(guard-against-mutually-exclusive-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.utf8")
`((locale ,identity #:default "en_US.UTF-8")
(after_name ,identity)
(field ,identity #:multi-value
#:default ("version" "synopsis"))
@ -170,7 +170,7 @@
(let ((parsed-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.utf8")))))
`((locale ,identity #:default "en_US.UTF-8")))))
(render-revision-package-version mime-types
conn
commit-hash
@ -308,7 +308,7 @@
(let ((parsed-query-parameters
(parse-query-parameters
request
`((locale ,identity #:default "en_US.utf8")
`((locale ,identity #:default "en_US.UTF-8")
(package_query ,identity)
(linter ,identity #:multi-value)
(message_query ,identity)

8
guix-data-service/web/revision/html.scm
View file

@ -725,13 +725,13 @@
path-base header-link
header-text)
(define total-package-synopsis
(assoc-ref package-synopsis-counts "en_US.utf8"))
(assoc-ref package-synopsis-counts "en_US.UTF-8"))
(define total-package-descriptions
(assoc-ref package-description-counts "en_US.utf8"))
(assoc-ref package-description-counts "en_US.UTF-8"))
(assoc-remove! package-synopsis-counts "en_US.utf8")
(assoc-remove! package-description-counts "en_US.utf8")
(assoc-remove! package-synopsis-counts "en_US.UTF-8")
(assoc-remove! package-description-counts "en_US.UTF-8")
(define synopsis-percentages
(map

2
scripts/guix-data-service-process-job.in
View file

@ -31,7 +31,7 @@
(setvbuf (current-output-port) 'line)
(setvbuf (current-error-port) 'line)
(setlocale LC_ALL "en_US.utf8")
(setlocale LC_ALL "en_US.UTF-8")
(debug-set! width 256)

11
sqitch/deploy/change_locale_values.sql Normal file
View file

@ -0,0 +1,11 @@
-- Deploy guix-data-service:change_locale_values to pg
BEGIN;
UPDATE lint_checker_descriptions SET locale = REPLACE(locale, '.utf8', '.UTF-8');
UPDATE package_metadata_tsvectors SET locale = REPLACE(locale, '.utf8', '.UTF-8');
UPDATE package_descriptions SET locale = REPLACE(locale, '.utf8', '.UTF-8');
UPDATE package_synopsis SET locale = REPLACE(locale, '.utf8', '.UTF-8');
UPDATE lint_warning_messages SET locale = REPLACE(locale, '.utf8', '.UTF-8');
COMMIT;

7
sqitch/revert/change_locale_values.sql Normal file
View file

@ -0,0 +1,7 @@
-- Revert guix-data-service:change_locale_values from pg
BEGIN;
-- XXX Add DDLs here.
COMMIT;

1
sqitch/sqitch.plan
View file

@ -68,3 +68,4 @@ builds_remove_build_server_id_derivation_unique_constraint 2020-06-30T20:17:48Z
derivation_output_details_sets_derivation_output_details_ids_index 2020-07-04T07:56:49Z Christopher Baines <mail@cbaines.net> # Add GIN index on derivation_output_details_ids
add-tsvectors-per-locale 2020-07-16T18:19:44Z daniela <daniela@linux-ijv5> # Add tsvectors per locale
change_package_descriptions_index 2020-08-23T11:56:00Z Christopher Baines <mail@cbaines.net> # Change the package_descriptions index
change_locale_values 2020-09-26T10:20:22Z Christopher Baines <mail@cbaines.net> # Change locale values

7
sqitch/verify/change_locale_values.sql Normal file
View file

@ -0,0 +1,7 @@
-- Verify guix-data-service:change_locale_values on pg
BEGIN;
-- XXX Add verifications here.
ROLLBACK;

4
tests/model-package-metadata.scm
View file

@ -43,8 +43,8 @@
((guix-data-service model package-metadata)
inferior-packages->translated-package-descriptions-and-synopsis
(lambda (inferior inferior-package)
(cons `(("en_US.utf8" . "Fake synopsis"))
`(("en_US.utf8" . "Fake description")))))
(cons `(("en_US.UTF-8" . "Fake synopsis"))
`(("en_US.UTF-8" . "Fake description")))))
(with-mock-inferior-packages
(lambda ()
(use-modules (guix-data-service model package)

4
tests/model-package.scm
View file

@ -49,8 +49,8 @@
((guix-data-service model package-metadata)
inferior-packages->translated-package-descriptions-and-synopsis
(lambda (inferior inferior-package)
(cons `(("en_US.utf8" . "Fake synopsis"))
`(("en_US.utf8" . "Fake description")))))
(cons `(("en_US.UTF-8" . "Fake synopsis"))
`(("en_US.UTF-8" . "Fake description")))))
(with-mock-inferior-packages
(lambda ()
(use-modules (guix-data-service model package)