From 7496065e8bdd7ac86d096a6cc2fcee4249115a43 Mon Sep 17 00:00:00 2001 From: David Galloway Date: Fri, 13 Mar 2020 17:54:13 -0400 Subject: [PATCH] scripts: Support new directory structure in signing scripts Sage requested a new directory structure on download.ceph.com to make posting new versions and potentially pulling releases easier. Instead of having all packages in a single repo under a `rpm-octopus` directory, `rpm-octopus` on d.c.c will symlink to `rpm-X.X.X` where X.X.X is the latest release. Signed-off-by: David Galloway --- scripts/sign-rpms | 116 +++++++++++++++++++++++----------------------- scripts/sync-pull | 10 ++-- scripts/sync-push | 46 +++++++++++++++--- 3 files changed, 105 insertions(+), 67 deletions(-) diff --git a/scripts/sign-rpms b/scripts/sign-rpms index b757fe5c..0fc7e9dc 100644 --- a/scripts/sign-rpms +++ b/scripts/sign-rpms @@ -4,6 +4,8 @@ # a repository layout. The layout follows this convention: # # /opt/repos/$project/$release/$distro/$distro_version +# OR (for octopus and later) +# /opt/repos/$project/$release-X.X.X/$distro/$distro_version # # If no arguments are passed in, all defined releases are used. It can # optionally be just one or any combination of them, like: @@ -17,80 +19,78 @@ keyid=460F3994 -# default releases -releases=${*:-"giant hammer infernalis testing"} +if [ $# -eq 0 ]; then + # Default releases if no arguments passed + releases=( mimic nautilus octopus ) +else + releases=( "$@" ) +fi # distros are not configurable. "rhel" might not exist in every release (for # example it doesn't exist for infernalis releases. distros=( centos rhel ) -# Although upstream these might be "el6" or "el7", we just use these since they +# Although upstream these might be "el7" or "el8", we just use these since they # are the same values used by the build system. -distro_versions=( 6 7 8 ) +distro_versions=( 7 8 ) # To unlock the gpg keys for the current run, it is requested over STDIN as # a password and later passed into GPG directly as a variable. read -s -p "Key Passphrase: " GPG_PASSPHRASE echo -for release in "${releases[@]}" - do - for distro in "${distros[@]}" - do - for distro_version in "${distro_versions[@]}" - do - path="/opt/repos/ceph/$release/$distro/$distro_version" - printf '=%.0s' {1..100} - echo "Release: " $release - echo "Distro: " $distro - echo "Distro Version: " $distro_version - echo "Path: " $path - printf '=%.0s' {1..100} - update_repo=0 - cd $path - for rpm in `find -name "*.rpm"` - do - # this call to `rpm -qi -p` will spit out metatada information - # from an rpm file which will tell us about the signature. This - # is significantly faster than letting gpg see if this needs to - # be signed or not. - signature=$(rpm -qi -p $rpm 2>/dev/null | grep ^Signature) - if ! grep -iq $keyid <<< "$signature" ; then - rpm_path=`readlink -f $rpm` - echo "signing: $rpm_path" - update_repo=1 +for release in "${releases[@]}"; do + for distro in "${distros[@]}"; do + for distro_version in "${distro_versions[@]}"; do + for path in /opt/repos/ceph/$release*; do + if [ -d "$path/$distro/$distro_version" ]; then + #path="/opt/repos/ceph/$release/$distro/$distro_version" + echo "Checking packages in: $path/$distro/$distro_version" + update_repo=0 + cd $path/$distro/$distro_version - echo "yes" | setsid rpm \ - --define "_gpg_name '$keyid'" \ - --define '_signature gpg' \ - --define '__gpg_check_password_cmd /bin/true' \ - --define "__gpg_sign_cmd %{__gpg} gpg --no-tty --yes --batch --no-armor --passphrase '$GPG_PASSPHRASE' --no-secmem-warning -u "%{_gpg_name}" --sign --detach-sign --output %{__signature_filename} %{__plaintext_filename}" \ - --resign "$rpm_path" + for rpm in `find -name "*.rpm"`; do + # this call to `rpm -qi -p` will spit out metatada information + # from an rpm file which will tell us about the signature. This + # is significantly faster than letting gpg see if this needs to + # be signed or not. + signature=$(rpm -qi -p $rpm 2>/dev/null | grep ^Signature) + if ! grep -iq $keyid <<< "$signature" ; then + rpm_path=`readlink -f $rpm` + echo "signing: $rpm_path" + update_repo=1 - fi - done + echo "yes" | setsid rpm \ + --define "_gpg_name '$keyid'" \ + --define '_signature gpg' \ + --define '__gpg_check_password_cmd /bin/true' \ + --define "__gpg_sign_cmd %{__gpg} gpg --no-tty --yes --batch --no-armor --passphrase '$GPG_PASSPHRASE' --no-secmem-warning -u "%{_gpg_name}" --sign --detach-sign --output %{__signature_filename} %{__plaintext_filename}" \ + --resign "$rpm_path" + + fi + done - # now sign the repomd.xml files - if [[ $update_repo -eq 1 ]]; then - for repomd in `find -name repomd.xml` - do - echo "signing repomd: $repomd" - gpg --batch --yes --passphrase "$GPG_PASSPHRASE" --detach-sign --armor -u $keyid $repomd - done - fi + # now sign the repomd.xml files + if [[ $update_repo -eq 1 ]]; then + for repomd in `find -name repomd.xml`; do + echo "signing repomd: $repomd" + gpg --batch --yes --passphrase "$GPG_PASSPHRASE" --detach-sign --armor -u $keyid $repomd + done + fi - # finally, update the repo metadata because when signing the RPM files - # the metadata gets all messed up. - if [[ $update_repo -eq 1 ]]; then - for directory in `ls $path` - do - cd $directory - # use the --no-database to workaround the large dbg packages issues - # https://tracker.ceph.com/issues/39387 - createrepo --no-database . - cd - - done - fi + # finally, update the repo metadata + if [[ $update_repo -eq 1 ]]; then + for directory in $(ls $path/$distro/$distro_version); do + cd $directory + # use the --no-database to workaround the large dbg packages issues + # https://tracker.ceph.com/issues/39387 + createrepo --no-database . + cd - + done + fi + + fi + done done done done diff --git a/scripts/sync-pull b/scripts/sync-pull index cdeb197c..8230cd83 100644 --- a/scripts/sync-pull +++ b/scripts/sync-pull @@ -38,14 +38,18 @@ for endpoint in https://chacra.ceph.com/repos/$project/$release/$sha1/$highest_c fi done +# Get numerical version number and create directory tree +version=$(echo $chacra_repo_status | python -mjson.tool | grep \"version\" | awk '{ print $2 }' | sed 's/"//g') +[[ -d /opt/repos/$project/$release-$version ]] | mkdir -p /opt/repos/$project/$release-$version/{debian/jessie,centos/{7,8}} + # Replace $highest_combo with your own DISTRO/VERSION if you don't want to sync from the repo with the most packages. -deb_cmd="ubuntu@chacra.ceph.com:/opt/repos/$project/$release/$sha1/$highest_combo/flavors/default/* /opt/repos/$project/$release/debian/jessie/" +deb_cmd="ubuntu@chacra.ceph.com:/opt/repos/$project/$release/$sha1/$highest_combo/flavors/default/* /opt/repos/$project/$release-$version/debian/jessie/" echo $deb_cmd echo "--------------------------------------------" -rsync -Lavh -e 'ssh -p 2222' --progress $deb_cmd +rsync -Lavh -e 'ssh -p 2222' --progress --exclude '*lockfile*' $deb_cmd for el_version in 7 8; do - el_cmd="ubuntu@chacra.ceph.com:/opt/repos/$project/$release/$sha1/centos/$el_version/flavors/default/* /opt/repos/$project/$release/centos/$el_version/" + el_cmd="ubuntu@chacra.ceph.com:/opt/repos/$project/$release/$sha1/centos/$el_version/flavors/default/* /opt/repos/$project/$release-$version/centos/$el_version/" echo $el_cmd echo "--------------------------------------------" rsync -Lavh -e 'ssh -p 2222' --progress $el_cmd diff --git a/scripts/sync-push b/scripts/sync-push index 1b677424..82cac939 100644 --- a/scripts/sync-push +++ b/scripts/sync-push @@ -8,17 +8,51 @@ # Since the binaries are created with a different repository layout, this # script maps directories like "centos/6" to "rpm-$release/el6" -releases=${*:-"firefly giant hammer infernalis testing"} +releases=${*:-"nautilus octopus"} ceph_sync() { release=$1 - deb_cmd="/opt/repos/ceph/$release/debian/jessie/* signer@download.ceph.com:/data/download.ceph.com/www/debian-$release/" - rsync --progress --exclude '*lockfile*' -avr $deb_cmd + for path in /opt/repos/ceph/$release*; do - for el_version in 7 8; do - el_cmd="/opt/repos/ceph/$release/centos/$el_version/* signer@download.ceph.com:/data/download.ceph.com/www/rpm-$release/el${el_version}/" - rsync --progress -avr $el_cmd + # We started using the new /opt/repos/ceph/$release-X.X.X format with Octopus. + # Older releases have all packages in one big $release dir without a dash. + if echo $path | grep -s '-'; then + # Octopus and later + newgen=true + else + # Nautilus and older + newgen=false + fi + + if $newgen; then + version=$(echo $path | cut -d '-' -f2) + ssh signer@download.ceph.com "mkdir -p /data/download.ceph.com/www/debian-$version" + + deb_cmd="$path/debian/jessie/* signer@download.ceph.com:/data/download.ceph.com/www/debian-$version/" + else + deb_cmd="$path/debian/jessie/* signer@download.ceph.com:/data/download.ceph.com/www/debian-$release/" + fi + rsync --progress --exclude '*lockfile*' -avr $deb_cmd + + for el_version in 7 8; do + if $newgen; then + ssh signer@download.ceph.com "mkdir -p /data/download.ceph.com/www/rpm-$version/el$el_version" + + el_cmd="$path/centos/$el_version/* signer@download.ceph.com:/data/download.ceph.com/www/rpm-$version/el${el_version}/" + else + el_cmd="$path/centos/$el_version/* signer@download.ceph.com:/data/download.ceph.com/www/rpm-$release/el${el_version}/" + fi + if [ -d "$path/centos/$el_version" ]; then + rsync --progress -avr $el_cmd + fi + done done + + # Since paths are listed alphabetically/numerically in the first `for` loop, the last $version is what gets used for the new symlink below. + if $newgen; then + ssh signer@download.ceph.com "ln -sfn /data/download.ceph.com/www/debian-$version /data/download.ceph.com/www/debian-$release; \ + ln -sfn /data/download.ceph.com/www/rpm-$version /data/download.ceph.com/www/rpm-$release" + fi } for i in "${releases[@]}" -- 2.39.5