Added support for multiple debian repositories

Option -d now requires an argument specifying which repo to publish to
This closes #42
This commit is contained in:
Andrew S. Rightenburg 2024-10-02 17:30:05 +08:00
parent b2e2d91d8b
commit f22d62bd27
Signed by: rail5
GPG key ID: BD8796E9783D5764
3 changed files with 216 additions and 85 deletions

View file

@ -98,7 +98,42 @@ function setup_build_farm() {
}
function get_list_of_debian_repos() {
find ${local_storage_directory:?}/repo/* -maxdepth 0 -type d 2>/dev/null
}
function delete_debian_repository() {
if [[ $# != 1 ]]; then
echo "bag args to create_new_debian_repository"; exit 1
fi
local repo_path="$1"
if [[ "$(dirname "$(realpath "$repo_path")")" != "${local_storage_directory:?}/repo" ]]; then
echo "Invalid path supplied to create_new_debian_repository"; exit 1
fi
dialog --title "$window_title" \
--yesno "Are you sure you would like to DELETE '$(basename "$repo_path")'?" 15 55
case $? in
0)
rm -rf "${repo_path:?}/"
return ;;
1)
return ;;
esac
}
function create_new_debian_repository() {
if [[ $# != 1 ]]; then
echo "bag args to create_new_debian_repository"; exit 1
fi
local repo_path="$1"
if [[ "$(dirname "$(realpath "$repo_path")")" != "${local_storage_directory:?}/repo" ]]; then
echo "Invalid path supplied to create_new_debian_repository"; exit 1
fi
# Ask the user for the name and email to use for the package signing key
local user_email="" user_name="" signing_key_fingerprint="" repo_url="" repo_friendlyname="" default_index_html_code="" repo_will_be_ghpages=false
{ user_email="$(dialog --title "$window_title" \
@ -113,27 +148,12 @@ function create_new_debian_repository() {
# Check if we've already made a key with that email
if gpg --list-secret-keys "$user_email"; then
key_exists_already=true
user_name=$(gpg --with-colons -k "$user_email" | awk -F: '$1=="uid" {print $10}' | sed 's/<.\+>//')
else
key_exists_already=false
fi
use_old_key=$key_exists_already
if [[ $key_exists_already == true ]]; then
dialog --title "$window_title" \
--yesno "A signing key already exists under '$user_email'.\n\nWould you like to DELETE this one and create a new signing key?" 15 55
case $? in
0)
use_old_key=false
;;
1)
use_old_key=true
user_name=$(gpg --with-colons -k "$user_email" | awk -F: '$1=="uid" {print $10}' | sed 's/<.\+>//')
;;
esac
fi
if [[ $use_old_key == false ]]; then
if [[ $key_exists_already == false ]]; then
{ user_name="$(dialog --title "$window_title" \
--inputbox "Input the NAME to be associated with the package GPG signing key" 15 55 \
2>&1 1>&3 3>&- )"; } 3>&1
@ -142,13 +162,6 @@ function create_new_debian_repository() {
# User pressed cancel
return
fi
if [[ $key_exists_already == true ]]; then
# Delete the old key before creating the new key
old_fingerprint=$(gpg -K --with-colons "$user_email" | awk -F: '$1=="fpr" {print $10}' | head -n 1)
gpg --batch --yes --delete-secret-keys "$old_fingerprint"
gpg --batch --yes --delete-keys "$user_email"
fi
# Create the new key
gpg --batch --gen-key <<EOF
@ -168,11 +181,11 @@ EOF
signing_key_fingerprint=$(gpg -K --with-colons "$user_email" | awk -F: '$1=="fpr" {print $10}' | head -n 1)
# Delete any old repository which may or may not exist
rm -rf "${local_storage_directory:?}/repo/"
rm -rf "${repo_path:?}/"
# Prepare the new repository for reprepro
mkdir -p "${local_storage_directory:?}/repo/conf"
cat > "${local_storage_directory:?}/repo/conf/distributions" <<EOF
mkdir -p "${repo_path:?}/conf"
cat > "${repo_path:?}/conf/distributions" <<EOF
Origin: $user_name
Label: $user_name
Codename: unstable
@ -200,10 +213,10 @@ EOF
repo_friendlyname=$(sed 's/[^A-Za-z0-9\-]//g' <<<"$user_name" | tr '[:upper:]' '[:lower:]')
# Export the public key to a file
gpg --export "$signing_key_fingerprint" > "${local_storage_directory:?}/repo/${repo_friendlyname:?}-signing-key.gpg"
gpg --export "$signing_key_fingerprint" > "${repo_path:?}/${repo_friendlyname:?}-signing-key.gpg"
# Create the repo .list file
cat > "${local_storage_directory:?}/repo/${repo_friendlyname:?}.list" <<EOF
cat > "${repo_path:?}/${repo_friendlyname:?}.list" <<EOF
deb $repo_url unstable main
deb-src $repo_url unstable main
EOF
@ -213,7 +226,7 @@ EOF
default_index_html_code="${default_index_html_code//\%REPO_FRIENDLYNAME\%/"$repo_friendlyname"}"
default_index_html_code="${default_index_html_code//\%REPO_URL\%/"$repo_url"}"
cat > "${local_storage_directory:?}/repo/index.html" <<<"$default_index_html_code"
cat > "${repo_path:?}/index.html" <<<"$default_index_html_code"
# Will this be some web server, or GitHub Pages?
# If it's GitHub Pages, we'll have to push changes etc rather than just change files
@ -229,7 +242,7 @@ EOF
;;
esac
local repo_conf_file="${local_storage_directory:?}/repo/autobuild_repo.conf"
local repo_conf_file="${repo_path:?}/autobuild_repo.conf"
echo "[repo]" > "${repo_conf_file:?}"
@ -263,7 +276,7 @@ EOF
echo "ghpages_url = \"$ghpages_url\"" >> "${repo_conf_file:?}"
# Initialize the git repo, set origin, and push
cd "${local_storage_directory:?}/repo" || (echo ""; echo "Confusing and fatal error in setting up GitHub Pages repository"; exit)
cd "${repo_path:?}" || (echo ""; echo "Confusing and fatal error in setting up GitHub Pages repository"; exit)
# Add our necessary credentials to the URL to push (https://user:pass@github.com/etc)
local push_url="${ghpages_url/:\/\//:\/\/$GITHUB_OWNER:$GITHUB_ACCESS_TOKEN@}"
@ -291,30 +304,107 @@ EOF
fi
dialog --title "$window_title" \
--infobox "Your Debian Repo is now configured\n\nYou can check the welcome page at $repo_url\nIf your repo is managed via GitHub Pages, it may take a few moments to publish." 15 55; sleep 7
--infobox "Your Debian Repo is now configured\n\nThe files are served in:\n$repo_path\n\nIf your repo is managed via GitHub Pages, it may take a few moments to publish." 15 55; sleep 7
}
function setup_debian_repo() {
# First, check: do we have one already?
debian_repository_already_exists=$(test -f /var/autobuild/repo/autobuild_repo.conf && echo "true" || echo "false")
if [[ $# != 1 ]]; then
echo "bad args to setup_debian_repo"; exit 1
fi
local repo_path="$1" selected_repo_is_ok=false selected_repo_exists_already="" new_repo_name=""
if [[ $debian_repository_already_exists == true ]]; then
# First, check: Are we either (1) making a new repository ('blank' input), or (2) modifying one which *actually* exists?
# In other words, refuse to continue if we've been asked to modify something which doesn't exist,
# Or which doesn't exist *inside* the proper directory.
if [[ "$repo_path" == "" ]]; then
# Creating a new repo, automatically ok
selected_repo_is_ok=true
else
for repository in $(get_list_of_debian_repos); do
if [[ "$repository" == "$repo_path" ]]; then
selected_repo_is_ok=true
break
fi
done
fi
if [[ "$selected_repo_is_ok" == false ]]; then
# Quit.
return
fi
# If we're making a new repository, let's get a name for it
if [[ "$repo_path" == "" ]]; then
# If this is the first repository, it must be called "default"
if [[ "$(get_list_of_debian_repos)" == "" ]]; then
new_repo_name="default"
else
{ new_repo_name="$(dialog --title "$window_title" \
--inputbox "Enter a name for this new repository:" 15 55 \
2>&1 1>&3 3>&- )"; } 3>&1
fi
case "$new_repo_name" in
("")
# User decided to cancel
return ;;
(*[!a-zA-Z0-9\-\_]*)
# Invalid characters
# Allowed characters: a-z, A-Z, 0-9, -, and _
dialog --title "$window_title" \
--infobox "Invalid repository name!\nRepo names may contain letters, numbers, hypens (-) and underscores (_)" 15 55;
sleep 7;
return ;;
(*)
# No problem. Move on
repo_path="$local_storage_directory/repo/$new_repo_name"
esac
fi
selected_repo_exists_already=$(test -f "${repo_path:?}/autobuild_repo.conf" && echo "true" || echo "false")
if [[ "$selected_repo_exists_already" == true ]]; then
dialog --title "$window_title" \
--yesno "It looks like a repository already exists in /var/autobuild/repo.\nWould you like to DELETE this repository and create a new one?" 15 55
--yesno "Selected existing repository: $(basename "$repo_path").\nWould you like to DELETE this repository?" 15 55
case $? in
0)
create_new_debian_repository
delete_debian_repository "$repo_path"
return ;;
1)
return ;;
esac
else
create_new_debian_repository
return
create_new_debian_repository "$repo_path"
fi
}
function setup_debian_repos() {
local next_page="" number_of_debian_repos="" menu_command_args="" iterator=1 selected_repo_path=""
# How many (and which) Debian Repos do we have set up already?
number_of_debian_repos="$(wc -l <<<"$(get_list_of_debian_repos)")"
menu_command_args=("$((number_of_debian_repos + 1))")
for repository in $(get_list_of_debian_repos); do
menu_command_args+=("$iterator" "$(basename "$repository")")
iterator=$((iterator + 1))
done
menu_command_args+=("$iterator" "Create New Repository")
{ next_page="$(dialog --title "$window_title" --menu "Debian Repositories" 15 55 "${menu_command_args[@]}" \
2>&1 1>&3 3>&- )"; } 3>&1
if [[ "$next_page" == "" ]]; then
return # User pressed cancel
fi
selected_repo_path="$(sed "$next_page"'q;d' <<<"$(get_list_of_debian_repos)")"
setup_debian_repo "$selected_repo_path" # Otherwise, get going
}
function clear_builds_directory() {
rm -rf "${local_storage_directory:?}/builds/"*
dialog --title "$window_title" \
@ -598,7 +688,7 @@ function setup_main_menu() {
5 \
1 "Edit config" \
2 "Install build farm" \
3 "Configure Debian repository" \
3 "Configure Debian repositories" \
4 "Clear 'builds' directory" \
5 "Exit" \
2>&1 1>&3 3>&- )"; } 3>&1 # Capture stderr output into next_page variable
@ -616,7 +706,7 @@ function setup_main_menu() {
setup_main_menu
;;
3)
setup_debian_repo
setup_debian_repos
setup_main_menu
;;
4)

View file

@ -22,8 +22,6 @@ pkgs_build_base_directory="builds/$(date +%Y-%h-%d-%H%M%S)"
CONFIG_FILE="$local_storage_directory/config.toml"
REPO_CONFIG_FILE="$local_storage_directory/repo/autobuild_repo.conf"
BUILDFARM_SCRIPTS_FILE="$autobuild_directory/build-farm/scripts/scripts.sh"
if [ ! -d "$local_storage_directory" ]; then
@ -86,6 +84,7 @@ packages_to_publish_to_repo=()
release_pages_to_make=()
make_github_release_pages=false
make_forgejo_release_pages=false
debian_repos_to_push_to=()
# Where is the daemon saving the built package files?
this_job_id="$(date +%Y-%h-%d.%H:%M:%S).$$"
@ -584,15 +583,40 @@ function push_forgejo_release_page() {
done
}
function parse_debian_repo_config() {
if [[ $# != 2 ]]; then
echo "bad args to prepare_git_debian_repo" && graceful_exit 1
fi
local repo_name="$1" requested_option="$2" repo_config_json="" repo_is_ghpages="" repo_ghpages_url="" output=""
repo_config_json="$(toml2json "$local_storage_directory/repo/${repo_name:?}/autobuild_repo.conf")"
case "$requested_option" in
"repo_is_ghpages")
output="$(jq -r ".repo.ghpages" <<<"$repo_config_json")"
;;
"repo_ghpages_url")
output="$(jq -r ".repo.ghpages_url" <<<"$repo_config_json")"
;;
esac
echo "$output"
}
function prepare_git_debian_repo() {
if [[ $REPO_IS_GHPAGES == "true" ]]; then
cd "$local_storage_directory/repo" || { echo "Could not cd into \"$local_storage_directory\""; echo "Have you configured your Debian repo properly in the CONFIG file?"; graceful_exit 1; }
if [[ $# != 1 ]]; then
echo "bad args to prepare_git_debian_repo" && graceful_exit 1
fi
local repo_name="$1" repo_is_ghpages="" repo_ghpages_url=""
repo_is_ghpages="$(parse_debian_repo_config "$repo_name" "repo_is_ghpages")"
repo_ghpages_url="$(parse_debian_repo_config "$repo_name" "repo_ghpages_url")"
if [[ $repo_is_ghpages == true ]]; then
cd "$local_storage_directory/repo/${repo_name:?}" || { echo "Could not cd into \"$local_storage_directory/repo/$repo_name\""; echo "Have you configured your Debian repo properly?"; graceful_exit 1; }
git config user.email "$GITHUB_EMAIL"
git config user.name "$GITHUB_OWNER"
# Add our necessary credentials to the URL to push (https://user:pass@github.com/etc)
local pull_url="${REPO_GHPAGES_URL/:\/\//:\/\/$GITHUB_OWNER:$GITHUB_ACCESS_TOKEN@}"
local pull_url="${repo_ghpages_url/:\/\//:\/\/$GITHUB_OWNER:$GITHUB_ACCESS_TOKEN@}"
git reset --hard
git pull "$pull_url" -q
@ -600,11 +624,17 @@ function prepare_git_debian_repo() {
}
function close_git_debian_repo() {
if [[ $REPO_IS_GHPAGES == "true" ]]; then
cd "$local_storage_directory/repo" || { echo "Could not cd into \"$local_storage_directory/repo\""; echo "Have you configured your Debian repo properly in the CONFIG file?"; graceful_exit 1; }
if [[ $# != 1 ]]; then
echo "bad args to close_git_debian_repo" && graceful_exit 1
fi
local repo_name="$1" repo_is_ghpages="" repo_ghpages_url=""
repo_is_ghpages="$(parse_debian_repo_config "$repo_name" "repo_is_ghpages")"
repo_ghpages_url="$(parse_debian_repo_config "$repo_name" "repo_ghpages_url")"
if [[ $repo_is_ghpages == "true" ]]; then
cd "$local_storage_directory/repo" || { echo "Could not cd into \"$local_storage_directory/repo/$repo_name\""; echo "Have you configured your Debian repo properly?"; graceful_exit 1; }
# Add our necessary credentials to the URL to push (https://user:pass@github.com/etc)
local push_url="${REPO_GHPAGES_URL/:\/\//:\/\/$GITHUB_OWNER:$GITHUB_ACCESS_TOKEN@}"
local push_url="${repo_ghpages_url/:\/\//:\/\/$GITHUB_OWNER:$GITHUB_ACCESS_TOKEN@}"
# Push any changes that we made before we called this function
git add --all
@ -618,25 +648,23 @@ function push_to_debian_repo() {
## If the Debian Repo in question is hosted as a Git Repository,
## Then this should be called AFTER prepare_git_debian_repo() and BEFORE close_git_debian_repo()
# FIXME: Allow for non-git-based Debian repos. Github pages debian repos are common but not universal
if [[ $# != 1 ]]; then
if [[ $# != 2 ]]; then
echo "bag args to push_to_debian_repo" && graceful_exit 5
fi
local PKGNAME="$1"
local PKGNAME="$1" repo_name="$2"
PKGNAME="$(basename "$PKGNAME")" # In case $PKGNAME is a file path to a .tar.gz archive
PKGNAME="${PKGNAME/.tar.gz/""}"
repo_base_directory=$(dirname "$(find "$local_storage_directory/repo" -name "pool" | head -n 1)")
repo_base_directory=$(dirname "$(find "$local_storage_directory/repo/${repo_name:?}" -name "pool" | head -n 1)")
if [[ "$repo_base_directory" == "." ]]; then
repo_base_directory="$local_storage_directory/repo" # Handle if 'pool' not found
repo_base_directory="$local_storage_directory/repo/${repo_name:?}" # Handle if 'pool' not found
fi
cd "$repo_base_directory" || { echo "Could not cd into \"$repo_base_directory\""; echo "Have you configured your Debian repo properly in the CONFIG file?"; graceful_exit 1; }
cd "$repo_base_directory" || { echo "Could not cd into \"$repo_base_directory\""; echo "Have you configured your Debian repo properly?"; graceful_exit 1; }
debian_distribution=$(grep Codename: <"$repo_base_directory/conf/distributions" | awk '{print $2}' | head -n 1)
@ -650,13 +678,15 @@ function maybe_publish_to_deb_repo() {
if [[ ${#packages_to_publish_to_repo[@]} -gt 0 ]]; then
echo "Publishing to repo..."
prepare_git_debian_repo
for pkg in "${packages_to_publish_to_repo[@]}"; do
push_to_debian_repo "$pkg"
for repository in "${debian_repos_to_push_to[@]}"; do
prepare_git_debian_repo "$repository"
for pkg in "${packages_to_publish_to_repo[@]}"; do
push_to_debian_repo "$pkg" "$repository"
done
close_git_debian_repo "$repository"
done
close_git_debian_repo
fi
}
@ -774,6 +804,7 @@ function display_help() {
echo " -d"
echo " --debian-repo"
echo " Distribute built packages to a Git-based Debian Repository managed with reprepro"
echo " Argument should be the name of a configured repo"
echo ""
echo " -g"
echo " --github-page"
@ -807,8 +838,8 @@ function display_help() {
echo " Write output to a provided file instead of standard output"
echo " e.g.: -L /tmp/some.log"
echo "Example:"
echo " autobuild -12dg -p liesel -p bookthief -p polonius"
echo " autobuild --i386 --arm64 --debian-repo --github-page --package liesel --package bookthief --package polonius"
echo " autobuild -12g -d default -p liesel -p bookthief -p polonius"
echo " autobuild --i386 --arm64 --debian-repo default --github-page --package liesel --package bookthief --package polonius"
echo " autobuild -p liesel -o ~/Desktop/"
}
@ -823,7 +854,7 @@ arguments=$(cat <&0) # Receive arguments from STDIN
eval set -- "$arguments"
TEMP=$(getopt -o 0123bdfghlL:no:p:r:su --long local,amd64,i386,arm64,bell,debian-repo,forgejo-page,github-page,help,list,log:,no-upgrade,output:,package:,remove-old-builds:,setup,upgrade \
TEMP=$(getopt -o 0123bd:fghlL:no:p:r:su --long local,amd64,i386,arm64,bell,debian-repo:,forgejo-page,github-page,help,list,log:,no-upgrade,output:,package:,remove-old-builds:,setup,upgrade \
-n 'autobuild' -- "$@")
if [ $? != 0 ] ; then echo "Terminating..." >&2 ; graceful_exit 1 ; fi
@ -837,7 +868,6 @@ building_locally=false
building_on_amd64=false
building_on_i386=false
building_on_arm64=false
debian_repo_is_configured=false
distribute_to_debian_repo=false
upgrading_vms=true
just_upgrading_vms=false
@ -876,15 +906,31 @@ while true; do
# Handled by the autobuild client
shift ;;
-d | --debian-repo )
if [[ -f "$REPO_CONFIG_FILE" ]]; then
debian_repo_is_configured=true
fi
if [[ $debian_repo_is_configured == "true" ]]; then
case "$2" in
("")
# None specified -- use default repo
this_debian_repo="default"
;;
(*[!a-zA-Z0-9\-\_]*)
# Invalid characters. Fail closed
# Allowed characters: a-z, A-Z, 0-9, -, and _
echo "Invalid Debian repository specified: $2"
graceful_exit 1
;;
(*)
# No problem. Move on
this_debian_repo="$2"
;;
esac
if [[ -f "$local_storage_directory/repo/$this_debian_repo/autobuild_repo.conf" ]]; then
distribute_to_debian_repo=true
debian_repos_to_push_to+=("$this_debian_repo")
else
echo "Debian Repository is not configured"
echo "Debian repository is not configured: $this_debian_repo"
graceful_exit 1
fi
shift ;;
shift 2 ;;
-f | --forgejo-page )
make_forgejo_release_pages=true; shift ;;
-g | --github-page )
@ -1005,16 +1051,6 @@ FORGEJO_OWNER=$(jq -r ".forgejo.repo_owner" <<<"$config_json")
### Check for Forgejo access token
FORGEJO_ACCESS_TOKEN=$(jq -r ".forgejo.access_token" <<<"$config_json")
# If the Debian Repo has been configured, parse that config file
if [[ $debian_repo_is_configured == "true" ]]; then
repo_config_json=$(toml2json "$REPO_CONFIG_FILE")
REPO_IS_GHPAGES=$(jq -r ".repo.ghpages" <<<"$repo_config_json")
REPO_GHPAGES_URL=$(jq -r ".repo.ghpages_url" <<<"$repo_config_json")
fi
##################################################
@ -1107,7 +1143,11 @@ fi
# Check if and how we're distributing the built packages
if [ $distribute_to_debian_repo == true ]; then
echo "Publishing packages to Debian repo"
echo -n "Publishing packages to Debian repo(s): "
for repository in "${debian_repos_to_push_to[@]}"; do
echo -n "$repository "
done
echo ""
for pkgname in "${packages_to_build[@]}"; do
packages_to_publish_to_repo+=("$pkgname")
done

View file

@ -7,7 +7,7 @@ autobuild \- Automatically build and distribute Debian packages
`autobuild --amd64 --github-page -p https://github.com/user/package.git`
`autobuild --i386 --debian-repo --forgejo-page -p /path/to/package.tar.gz`
`autobuild --i386 --debian-repo default --forgejo-page -p /path/to/package.tar.gz`
# DESCRIPTION
Autobuild automatically retrieves Debian source packages (via Git or local tar.gz archive), builds them on a virtual machine build-farm for multiple architectures, and distributes them. It can save the built package locally, distribute them to a GitHub/Forgejo Release Page, and distribute them to a Git-based Debian Repository with reprepro.
@ -96,6 +96,7 @@ The configuration file can be found at /var/autobuild/config.toml. This file may
Distribute built packages to a Git-based Debian Repository via
reprepro
This repository must be configured via the config file
The argument should be the name of a Debian repo to push to
# AUTHOR
rail5 (andrew@rail5.org)