diff --git a/README.md b/README.md index 4579d44..431ff6f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Scripts to ease the update of Ruby on Rails core packages for Fedora Project. ## Usage ``` -./railsbuild FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION +./railsbuild FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION ``` To update Rails from 4.1.4 to 4.1.5 for rawhide: @@ -29,41 +29,33 @@ build, but just let railsbuild to prepare the update. ### railsbuild-prepare ``` -./railsbuild-prepare FEDORA_VERSION +./railsbuild-prepare FEDORA_BRANCH ``` Prepares directories for the railsbuild scripts including fetching Fedora and upstream repositories at ~/.railsbuild/upstream/rails and -~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem. - -### railsbuild-fetch-tests - -``` -railsbuild-fetch-tests FEDORA_VERSION RAILS_VERSION -``` -Gets the upstream tests from the repository at ~/.railsbuild/upstream/rails -and move the gzipped test suites to Fedora git repositories at -~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem. +~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem. ### railsbuild-update-pkgs ``` -./railsbuild-update-pkgs FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION +./railsbuild-update-pkgs FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION ``` Prepares the update of all Rails gems at once in Fedora git repositories at: -~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem. +~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem. -### railsbuild-update-bootstrapped +### railsbuild-fetch-sources ``` -./railsbuild-update-bootstrapped FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION +railsbuild-fetch-sources FEDORA_BRANCH ``` -Prepare the update of previously bootstrapped Rails gems at once in Fedora git -repositories at: ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem. +Get the information about sources (test suites, js, ...) for Rails gems from +their spec files, prepare the `git archive` tarballs to Fedora git repositories +at ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem ### railsbuild-build ``` -./railsbuild-build FEDORA_VERSION RAILS_VERSION +./railsbuild-build FEDORA_BRANCH RAILS_VERSION ``` Does the builds of gems. It always tries to do a scratch-build first and if that goes fine, does a final build. Afterwards it waits for the build diff --git a/railsbuild b/railsbuild index c01ea03..2485d5f 100755 --- a/railsbuild +++ b/railsbuild @@ -6,7 +6,7 @@ # Automatic builds of Ruby on Rails core packages for Fedora Project # # Usage: -# ./railsbuild [-n] FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION +# ./railsbuild [-n] FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION # # -n Do not run builds. Useful for major version upgrades. # @@ -14,7 +14,7 @@ # railsbuild-common print_help () { - echo "Usage: ./railsbuild [-n] FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION" + echo "Usage: ./railsbuild [-n] FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION" } if [ "$1" = "--help" ]; then @@ -34,20 +34,20 @@ if [ "$3" = "" ] || [ "$2" = "" ] || [ "$1" = "" ]; then exit 1 fi -FEDORA_VERSION=$1 +FEDORA_BRANCH=$1 OLD_RAILS_VERSION=$2 NEW_RAILS_VERSION=$3 # Create directory structure and get the upstream repositories -$(dirname $(readlink -f "$0"))/railsbuild-prepare $FEDORA_VERSION - -# Get the test suites -$(dirname $(readlink -f "$0"))/railsbuild-fetch-tests $FEDORA_VERSION $NEW_RAILS_VERSION +$(dirname $(readlink -f "$0"))/railsbuild-prepare $FEDORA_BRANCH # Update specs -$(dirname $(readlink -f "$0"))/railsbuild-update-pkgs $FEDORA_VERSION $OLD_RAILS_VERSION $NEW_RAILS_VERSION +$(dirname $(readlink -f "$0"))/railsbuild-update-pkgs $FEDORA_BRANCH $OLD_RAILS_VERSION $NEW_RAILS_VERSION + +# Fetch sources +$(dirname $(readlink -f "$0"))/railsbuild-fetch-sources $FEDORA_BRANCH $OLD_RAILS_VERSION $NEW_RAILS_VERSION # Run the build if not opted-out [[ "$NO_BUILD" ]] || { - $(dirname $(readlink -f "$0"))/railsbuild-build $FEDORA_VERSION $NEW_RAILS_VERSION + $(dirname $(readlink -f "$0"))/railsbuild-build $FEDORA_BRANCH $NEW_RAILS_VERSION } diff --git a/railsbuild-build b/railsbuild-build index 252335f..52ec9e3 100755 --- a/railsbuild-build +++ b/railsbuild-build @@ -6,26 +6,20 @@ # this script after it failed will pick up the work from the last run. # # Usage: -# ./railsbuild-build FEDORA_VERSION VERSION +# ./railsbuild-build FEDORA_BRANCH VERSION . $(dirname $(readlink -f "$0"))/railsbuild-common -FEDORA_VERSION=$1 +FEDORA_BRANCH=$1 VERSION=$2 +# Check changelog in spec file for same EVR for gem in "${GEMS[@]}" do - pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem - # Skip already done builds - if [ -f .built ]; then - echo "$gem: build for $gem already done" - continue - fi - - # Check changelog in spec file for same EVR + pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem if [ -n "$( sed -n '/^%changelog/,$p' rubygem-$gem.spec \ - | grep -E " [0-9]+\:[0-9]+\.[0-9]+\.[0-9]+\-[0-9]+$" \ + | grep -E " ([0-9]+:)?[0-9]+\.[0-9]+\.[0-9]+-[0-9]+$" \ | head -n 2 | rev | cut -d' ' -f1 \ | sort | uniq -c | grep -vE "^[ ]*1 " )" ]; then @@ -33,15 +27,48 @@ do echo "$gem: same EVR in changelog already exists" exit 1 fi + popd +done + + +# Create side tag +if [[ -f ~/.railsbuild/$FEDORA_BRANCH/.side-tag ]]; then + echo -n "Reusing side tag: " +else + echo -n "Requesting new side tag: " + # Run this under the dist-gi repo to let fedpkg to figure the right base + # tag. + fedpkg --release $FEDORA_BRANCH request-side-tag \ + | sed -nr "/Side tag/ s/.*'(f[[:digit:]]+-build-side-[[:digit:]]+)'.*/\1/ p" > ~/.railsbuild/$FEDORA_BRANCH/.side-tag +fi +SIDE_TAG=`cat ~/.railsbuild/$FEDORA_BRANCH/.side-tag` +echo $SIDE_TAG + + +koji edit-sidetag --rpm-macro _with_bootstrap=1 $SIDE_TAG + + +# Build all gems in first round +for gem in "${GEMS[@]}" +do + pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem + # Skip already done builds + if [ -f .built ]; then + echo "$gem: build for $gem already done" + popd + continue + fi + rm .scratch-built rm *.src.rpm - fedpkg scratch-build --srpm + fedpkg scratch-build --target=$SIDE_TAG --srpm if [ "$?" = "0" ]; then touch .scratch-built - git stash echo "= SUCCESS ======" echo "$gem: scratch build of rubygem-$gem succeeded" + + git stash fedpkg import rubygem-$gem-*.src.rpm > /dev/null if [ ! "$?" = "0" ]; then @@ -50,15 +77,21 @@ do exit 1 fi - git commit -am "Update to $gem $VERSION" + fedpkg clog + sed -i 's/^\s*//' clog + + git commit -F clog git push - fedpkg build + + PACKAGE_NVR=$(fedpkg verrel)$(test -f .bootstrapped && cat .bootstrapped) + + fedpkg build --target=$SIDE_TAG if [ "$?" = "0" ]; then echo "= SUCCESS ======" echo "$gem: build of rubygem-$gem succeeded" echo "$gem: waiting until the build becomes available..." - koji wait-repo f$FEDORA_VERSION-build --build=rubygem-$gem-$VERSION-1.fc$FEDORA_VERSION + koji wait-repo $SIDE_TAG --request --build=$PACKAGE_NVR touch .built else echo "= FAIL ======" @@ -74,45 +107,39 @@ do popd done -# Update bootsrapped gems -$(dirname $(readlink -f "$0"))/railsbuild-update-bootstrapped $FEDORA_VERSION $VERSION -# Build bootsrapped gems again +koji edit-sidetag --remove-rpm-macro _with_bootstrap $SIDE_TAG + + +# Build bootstrapped gems again for gem in "${BOOTSTRAPPED_GEMS[@]}" do - pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem + pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem # Skip already done builds if [ ! -f .bootstrapped ]; then - echo "$gem: final build for bootsrapped $gem already done" + echo "$gem: final build for bootstrapped $gem already done" + popd continue fi + rm .scratch-built rm *.src.rpm - fedpkg scratch-build --srpm + fedpkg scratch-build --target=$SIDE_TAG --srpm if [ "$?" = "0" ]; then touch .scratch-built - git stash echo "= SUCCESS ======" echo "$gem: scratch build of rubygem-$gem succeeded" - fedpkg import rubygem-$gem-*.src.rpm > /dev/null - if [ ! "$?" = "0" ]; then - echo "= FAIL ======" - echo "$gem: fedpkg import failed" - exit 1 - fi + PACKAGE_NVR=$(fedpkg verrel) - git commit -am "Enable tests" - git push - fedpkg build + fedpkg build --target=$SIDE_TAG if [ "$?" = "0" ]; then echo "= SUCCESS ======" echo "$gem: build of rubygem-$gem succeeded" echo "$gem: waiting until the build becomes available..." - koji wait-repo f$FEDORA_VERSION-build --build=rubygem-$gem-$VERSION-1.fc$FEDORA_VERSION - rm .bootsrapped + koji wait-repo $SIDE_TAG --request --build=$PACKAGE_NVR else echo "= FAIL ======" echo "$gem: build of rubygem-$gem failed" @@ -130,7 +157,9 @@ done echo echo echo "= SUCCESSFULLY DONE =====" -echo "Build of Ruby on Rails $VERSION for Fedora release $FEDORA_VERSION done." +echo "Build of Ruby on Rails $VERSION for Fedora release $FEDORA_BRANCH done." +echo +echo "Please submit for $SIDE_TAG via Bodhi." echo echo "Run the following to delete the build metadata:" -echo " rm ~/.railsbuild/f$FEDORA_VERSION/rubygem-*/.*built" +echo " rm ~/.railsbuild/$FEDORA_BRANCH/rubygem-*/.*built" diff --git a/railsbuild-common b/railsbuild-common index 83fde80..85dd776 100644 --- a/railsbuild-common +++ b/railsbuild-common @@ -3,20 +3,23 @@ RAILSBUILD_VERSION=0.4 -# Fedora master branch -FEDORA_RAWHIDE=24 - # Ruby on Rails gems GEMS=(activesupport activejob activemodel - activerecord - actionview actionpack + actionview + activerecord actionmailer + actioncable railties + activestorage + actionmailbox + actiontext rails) # Gems that needs to be rebuild without bootstrapping -BOOTSTRAPPED_GEMS=(actionview - actionpack) +BOOTSTRAPPED_GEMS=(activemodel + actionpack + actionview + railties) diff --git a/railsbuild-fetch-sources b/railsbuild-fetch-sources new file mode 100755 index 0000000..e6269e9 --- /dev/null +++ b/railsbuild-fetch-sources @@ -0,0 +1,59 @@ +#!/bin/bash +# Get the information about sources (test suites, js, ...) for Rails gems from +# their spec files, prepare the `git archive` tarballs from +# ~/.railsbuild/upstream/rails (upstream git repo) and place them in +# ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem +# +# Usage: +# ./railsbuild-fetch-tests FEDORA_BRANCH + +. $(dirname $(readlink -f "$0"))/railsbuild-common + +FEDORA_BRANCH=$1 + +cd ~/.railsbuild/upstream/rails +git pull + +for gem in "${GEMS[@]}" +do + echo "Getting sources for $gem..." + + VERSION=`cat ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/rubygem-$gem.spec | sed -rn "/^Version: / s/^Version: (.*)$/\1/ p"` + GEM_FILE="$gem-$VERSION.gem" + + # Is .gem already in place? + if [[ -f ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/$GEM_FILE ]]; then + echo " ✔ $GEM_FILE already exists" + else + pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/ >> /dev/null + gem fetch $gem --version $VERSION + popd >> /dev/null + fi + + GIT_ARCHIVE_CMDS=`grep -R -B1 '^Source[[:digit:]]*: ' ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/rubygem-$gem.spec | \ + sed -nr "/^#/ s/.*(git archive.*$gem.*\.tar\.gz.*)/\1/ p"` + + if [[ -z "$GIT_ARCHIVE_CMDS" ]]; then + echo " ⚠ No additional source identified" + continue + fi + + echo "$GIT_ARCHIVE_CMDS" | while read GIT_ARCHIVE_CMD; do + TARBALL=`echo $GIT_ARCHIVE_CMD | sed -r "s/.*($gem.*\.tar.\gz).*/\1/"` + + # Is source already in place? + if [[ -f ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/$TARBALL ]]; then + echo " ✔ $TARBALL already exists" + continue + fi + + cd ~/.railsbuild/upstream/rails/$gem + + `$GIT_ARCHIVE_CMD` + + mv $TARBALL ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/. + + echo " $TARBALL created" + done +done + diff --git a/railsbuild-fetch-tests b/railsbuild-fetch-tests deleted file mode 100755 index 3ad0daf..0000000 --- a/railsbuild-fetch-tests +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Get the test suites for Rails gems from ~/.railsbuild/upstream/rails (upstream git repo) -# and place them in ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem -# -# Usage: -# ./railsbuild-fetch-tests FEDORA_VERSION VERSION - -. $(dirname $(readlink -f "$0"))/railsbuild-common - -FEDORA_VERSION=$1 -VERSION=$2 - -cd ~/.railsbuild/upstream/rails -git pull -git checkout v$VERSION - -for gem in "${GEMS[@]}" -do - echo "Getting test suite for $gem..." - - UPDATED=`ls ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem/ | grep "$gem-$VERSION-tests.tgz"` - - # Are files already in place? - if [ "$UPDATED" == "$gem-$VERSION-tests.tgz" ]; then - continue - fi - - cd ~/.railsbuild/upstream/rails/$gem - tar czvf $gem-$VERSION-tests.tgz test/ - cp $gem-$VERSION-tests.tgz ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem/. -done - diff --git a/railsbuild-prepare b/railsbuild-prepare index 79db3ac..0cea146 100755 --- a/railsbuild-prepare +++ b/railsbuild-prepare @@ -2,24 +2,24 @@ # Prepare directories for the railsbuild scripts including fetching Fedora # and upstream repositories at: # ~/.railsbuild/upstream/rails -# ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem +# ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem # # Usage: -# ./railsbuild-prepare FEDORA_VERSION +# ./railsbuild-prepare FEDORA_BRANCH . $(dirname $(readlink -f "$0"))/railsbuild-common -FEDORA_VERSION=$1 +FEDORA_BRANCH=$1 for gem in "${GEMS[@]}" do - dir=~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem + dir=~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem if [ -d $dir ]; then echo "$dir exists" else echo "Creating $dir" - mkdir -p ~/.railsbuild/f$FEDORA_VERSION/ - pushd ~/.railsbuild/f$FEDORA_VERSION/ + mkdir -p ~/.railsbuild/$FEDORA_BRANCH/ + pushd ~/.railsbuild/$FEDORA_BRANCH/ fedpkg co rubygem-$gem popd fi diff --git a/railsbuild-update-bootstrapped b/railsbuild-update-bootstrapped deleted file mode 100755 index 69bbf52..0000000 --- a/railsbuild-update-bootstrapped +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Prepare the update of previously bootstrapped Rails gems at once in Fedora git -# repositories at: -# ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem -# -# Usage: -# ./railsbuild-update-boostrapped FEDORA_VERSION NEW_VERSION - -. $(dirname $(readlink -f "$0"))/railsbuild-common - -FEDORA_VERSION=$1 -NEW_VERSION=$2 - -for gem in "${BOOTSTRAPPED_GEMS[@]}" -do - echo "Building previously bootstrapped $gem:" - pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem/ - - EPOCH=`cat ./rubygem-$gem.spec | grep -E "Epoch:(.*)" | sed 's/Epoch: //'` - if [ "$EPOCH" ]; then - epoch_prefix="$EPOCH:" - else - epoch_prefix="" - fi - - # Disable bootstrapping - sed -i -e "s/%global bootstrap 1/%global bootstrap 0/" rubygem-$gem.spec - sed -i -e "s/Release:.*/Release: 2%{?dist}/g" rubygem-$gem.spec - - # Set to default locale for the changelog date format - export LC_ALL=C - - CHANGELOG='* ' - CHANGELOG+=`date +'%a %b %d %Y'` - CHANGELOG+=" " - CHANGELOG+=`rpmdev-packager` - CHANGELOG+=" - $epoch_prefix$NEW_VERSION-2\n" - CHANGELOG+="- Enable tests" - - sed -i -e "s/%changelog/%changelog\n$CHANGELOG\n/g" rubygem-$gem.spec - - echo "$gem: rubygem-$gem.spec updated" - popd -done diff --git a/railsbuild-update-pkgs b/railsbuild-update-pkgs index a158d2f..f0a5b89 100755 --- a/railsbuild-update-pkgs +++ b/railsbuild-update-pkgs @@ -1,59 +1,62 @@ #!/bin/bash # Prepare the update of all Rails gems at once in Fedora git # repositories at: -# ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem +# ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem # # Usage: -# ./railsbuild-update-pkgs FEDORA_VERSION OLD_VERSION NEW_VERSION +# ./railsbuild-update-pkgs FEDORA_BRANCH OLD_VERSION NEW_VERSION . $(dirname $(readlink -f "$0"))/railsbuild-common -FEDORA_VERSION=$1 +FEDORA_BRANCH=$1 OLD_VERSION=$2 NEW_VERSION=$3 +# Fetch BZ ticket number created by the-new-hotness. +RAILS_RHBZ=$(curl -s "https://bugzilla.redhat.com/rest/bug?product=Fedora&component=rubygem-rails&bug_status=__open__&creator=upstream-release-monitoring%40fedoraproject.org&order=bug_id%20desc&include_fields=id" \ + | sed -r -e 's/.*\[(.*)\].*/\1/' -e 's/},/}\n/g' \ + | sed -r 's/.*:([[:digit:]]+).*/\1/' \ + | head -1 +) + for gem in "${GEMS[@]}" do echo "Updating $gem:" - pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem/ + pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/ git pull - if [ "$FEDORA_VERSION" = "$FEDORA_RAWHIDE" ]; then - git checkout master - else - git checkout f$FEDORA_VERSION - fi - gem fetch $gem --version $NEW_VERSION + git checkout $FEDORA_BRANCH UPDATED=`cat ./rubygem-$gem.spec | grep "Version: $NEW_VERSION"` if [ "$UPDATED" == "Version: $NEW_VERSION" ]; then echo "$gem: already updated" + popd continue fi - EPOCH=`cat ./rubygem-$gem.spec | grep -E "Epoch:(.*)" | sed 's/Epoch: //'` - if [ "$EPOCH" ]; then - epoch_prefix="$EPOCH:" + # Try to recycle old changelog entry to keep the formatting. + CHANGELOG=$(grep "^- .* $OLD_VERSION" rubygem-$gem.spec) + if [[ -z $CHANGELOG ]]; then + CHANGELOG="- Update to $gem $NEW_VERSION" else - epoch_prefix="" + CHANGELOG=$(echo $CHANGELOG | sed "s/$OLD_VERSION/$NEW_VERSION/") fi - sed -i -e "s/git checkout v$OLD_VERSION/git checkout v$NEW_VERSION/" rubygem-$gem.spec - sed -i -e "s/$OLD_VERSION-tests.tgz/$NEW_VERSION-tests.tgz/" rubygem-$gem.spec - sed -i -e "s/Version:.*$OLD_VERSION/Version: $NEW_VERSION/g" rubygem-$gem.spec - sed -i -e "s/Release:.*/Release: 1%{?dist}/g" rubygem-$gem.spec + # Include reference to BZ created by the-new-hotness. + if [[ -n $RAILS_RHBZ ]]; then + CHANGELOG+=$'\n' + if [[ $gem = "rails" ]]; then + CHANGELOG+=" Resolves: rhzb#$RAILS_RHBZ" + else + CHANGELOG+=" Related: rhzb#$RAILS_RHBZ" + fi + fi - # Set to default locale for the changelog date format - export LC_ALL=C + rpmdev-bumpspec -n "$NEW_VERSION" -c "$CHANGELOG" rubygem-$gem.spec - CHANGELOG='* ' - CHANGELOG+=`date +'%a %b %d %Y'` - CHANGELOG+=" " - CHANGELOG+=`rpmdev-packager` - CHANGELOG+=" - $epoch_prefix$NEW_VERSION-1\n" - CHANGELOG+="- Update to $gem $NEW_VERSION" + sed -i -e "/git archive/ s/$OLD_VERSION/$NEW_VERSION/g" rubygem-$gem.spec + sed -i -e "s/$OLD_VERSION-tests.tar.gz/$NEW_VERSION-tests.tar.gz/" rubygem-$gem.spec - sed -i -e "s/%changelog/%changelog\n$CHANGELOG\n/g" rubygem-$gem.spec echo "$gem: rubygem-$gem.spec updated" popd done @@ -61,9 +64,7 @@ done # Mark bootsrapping for gem in "${BOOTSTRAPPED_GEMS[@]}" do - pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem/ - # Enable bootstrapping if necessary - sed -i -e "s/%global bootstrap 0/%global bootstrap 1/" rubygem-$gem.spec - touch .bootstrapped + pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/ + echo "~bootstrap" > .bootstrapped popd done